2026-02-18T06:29:44.365 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-18T06:29:44.380 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-18T06:29:44.458 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909 branch: 20.2.1_2.17.26-baseline_2 description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '54909' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: 20.2.1_2.17.26-baseline_2 ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: zlib bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: 20.2.1_2.17.26-baseline_2 sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 7878 sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: 20.2.1_2.17.26-baseline_2 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed targets: trial006.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPP05jXG1JMiRsC5EKWKC4JiRMBDH/oEdWq9Zi4ru9TMCkTxM7/9hlIOEHzP+L3FV2XDq89oJpv4o0bMYCJe/V8= trial051.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXWP8acdBulI9g3phySvn7Yy8qF7TW7SBOxFGID5vjvI+2GKGhwlNx0Ib+pGTj2SVsuhiqbtgtA24CXqUWxQak= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-17_21:26:40 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682 2026-02-18T06:29:44.458 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa; will attempt to use it 2026-02-18T06:29:44.458 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa/tasks 2026-02-18T06:29:44.458 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-18T06:29:44.460 INFO:teuthology.task.internal:Checking packages... 2026-02-18T06:29:44.460 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'adc1923395f2bcd6ae89b5d331d278364d1848ed' 2026-02-18T06:29:44.460 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-18T06:29:44.460 INFO:teuthology.packaging:ref: None 2026-02-18T06:29:44.460 INFO:teuthology.packaging:tag: None 2026-02-18T06:29:44.460 INFO:teuthology.packaging:branch: 20.2.1_2.17.26-baseline_2 2026-02-18T06:29:44.460 INFO:teuthology.packaging:sha1: adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:29:44.460 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=20.2.1_2.17.26-baseline_2 2026-02-18T06:29:44.575 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-670.gadc19233 2026-02-18T06:29:44.577 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-18T06:29:44.586 INFO:teuthology.task.internal:no buildpackages task found 2026-02-18T06:29:44.586 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-18T06:29:44.613 INFO:teuthology.task.internal:Saving configuration 2026-02-18T06:29:44.628 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-18T06:29:44.637 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-18T06:29:44.713 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial006.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-18 06:26:19.597544', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPP05jXG1JMiRsC5EKWKC4JiRMBDH/oEdWq9Zi4ru9TMCkTxM7/9hlIOEHzP+L3FV2XDq89oJpv4o0bMYCJe/V8='} 2026-02-18T06:29:44.769 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial051.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-18 06:26:19.598489', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXWP8acdBulI9g3phySvn7Yy8qF7TW7SBOxFGID5vjvI+2GKGhwlNx0Ib+pGTj2SVsuhiqbtgtA24CXqUWxQak='} 2026-02-18T06:29:44.769 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-18T06:29:44.778 INFO:teuthology.task.internal:roles: ubuntu@trial006.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-18T06:29:44.779 INFO:teuthology.task.internal:roles: ubuntu@trial051.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-18T06:29:44.779 INFO:teuthology.run_tasks:Running task console_log... 2026-02-18T06:29:44.910 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x78b6037ba520>, signals=[15]) 2026-02-18T06:29:44.910 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-18T06:29:44.919 INFO:teuthology.task.internal:Opening connections... 2026-02-18T06:29:44.919 DEBUG:teuthology.task.internal:connecting to ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:29:44.920 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:29:44.982 DEBUG:teuthology.task.internal:connecting to ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:29:44.982 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial051.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:29:45.042 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-18T06:29:45.054 DEBUG:teuthology.orchestra.run.trial006:> uname -m 2026-02-18T06:29:45.069 INFO:teuthology.orchestra.run.trial006.stdout:x86_64 2026-02-18T06:29:45.069 DEBUG:teuthology.orchestra.run.trial006:> cat /etc/os-release 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:NAME="CentOS Stream" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:VERSION="9" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:ID="centos" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:ID_LIKE="rhel fedora" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:VERSION_ID="9" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:PLATFORM_ID="platform:el9" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:ANSI_COLOR="0;31" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:LOGO="fedora-logo-icon" 2026-02-18T06:29:45.124 INFO:teuthology.orchestra.run.trial006.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-18T06:29:45.125 INFO:teuthology.orchestra.run.trial006.stdout:HOME_URL="https://centos.org/" 2026-02-18T06:29:45.125 INFO:teuthology.orchestra.run.trial006.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-18T06:29:45.125 INFO:teuthology.orchestra.run.trial006.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-18T06:29:45.125 INFO:teuthology.orchestra.run.trial006.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-18T06:29:45.125 INFO:teuthology.lock.ops:Updating trial006.front.sepia.ceph.com on lock server 2026-02-18T06:29:45.198 DEBUG:teuthology.orchestra.run.trial051:> uname -m 2026-02-18T06:29:45.213 INFO:teuthology.orchestra.run.trial051.stdout:x86_64 2026-02-18T06:29:45.213 DEBUG:teuthology.orchestra.run.trial051:> cat /etc/os-release 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:NAME="CentOS Stream" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:VERSION="9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:ID="centos" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:ID_LIKE="rhel fedora" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:VERSION_ID="9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:PLATFORM_ID="platform:el9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:ANSI_COLOR="0;31" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:LOGO="fedora-logo-icon" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:HOME_URL="https://centos.org/" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-18T06:29:45.269 INFO:teuthology.orchestra.run.trial051.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-18T06:29:45.270 INFO:teuthology.lock.ops:Updating trial051.front.sepia.ceph.com on lock server 2026-02-18T06:29:45.344 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-18T06:29:45.359 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-18T06:29:45.370 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-18T06:29:45.370 DEBUG:teuthology.orchestra.run.trial006:> test '!' -e /home/ubuntu/cephtest 2026-02-18T06:29:45.372 DEBUG:teuthology.orchestra.run.trial051:> test '!' -e /home/ubuntu/cephtest 2026-02-18T06:29:45.387 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-18T06:29:45.397 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-18T06:29:45.397 DEBUG:teuthology.orchestra.run.trial006:> test -z $(ls -A /var/lib/ceph) 2026-02-18T06:29:45.427 DEBUG:teuthology.orchestra.run.trial051:> test -z $(ls -A /var/lib/ceph) 2026-02-18T06:29:45.443 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-18T06:29:45.596 INFO:teuthology.run_tasks:Running task kernel... 2026-02-18T06:29:45.615 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-18T06:29:45.615 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-18T06:29:45.615 DEBUG:teuthology.orchestra.run.trial006:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-18T06:29:45.616 DEBUG:teuthology.orchestra.run.trial051:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-18T06:29:45.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T06:29:45.630 DEBUG:teuthology.orchestra.run.trial051:> uname -r 2026-02-18T06:29:45.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T06:29:45.630 DEBUG:teuthology.orchestra.run.trial006:> uname -r 2026-02-18T06:29:45.686 INFO:teuthology.orchestra.run.trial051.stdout:5.14.0-645.el9.x86_64 2026-02-18T06:29:45.686 INFO:teuthology.task.kernel:Running kernel on trial051: 5.14.0-645.el9.x86_64 2026-02-18T06:29:45.686 DEBUG:teuthology.orchestra.run.trial051:> sudo yum install -y kernel 2026-02-18T06:29:45.687 INFO:teuthology.orchestra.run.trial006.stdout:5.14.0-645.el9.x86_64 2026-02-18T06:29:45.687 INFO:teuthology.task.kernel:Running kernel on trial006: 5.14.0-645.el9.x86_64 2026-02-18T06:29:45.687 DEBUG:teuthology.orchestra.run.trial006:> sudo yum install -y kernel 2026-02-18T06:29:49.686 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - BaseOS 2.4 MB/s | 8.9 MB 00:03 2026-02-18T06:29:51.445 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - BaseOS 1.6 MB/s | 8.9 MB 00:05 2026-02-18T06:29:52.882 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - AppStream 10 MB/s | 27 MB 00:02 2026-02-18T06:29:55.257 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - AppStream 8.2 MB/s | 27 MB 00:03 2026-02-18T06:29:55.655 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - CRB 20 MB/s | 7.8 MB 00:00 2026-02-18T06:29:56.523 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - Extras packages 188 kB/s | 20 kB 00:00 2026-02-18T06:29:57.355 INFO:teuthology.orchestra.run.trial051.stdout:Extra Packages for Enterprise Linux 27 MB/s | 20 MB 00:00 2026-02-18T06:30:00.409 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 7.8 MB 00:02 2026-02-18T06:30:00.498 INFO:teuthology.orchestra.run.trial051.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-18T06:30:01.510 INFO:teuthology.orchestra.run.trial051.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-18T06:30:01.533 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:30:01.534 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:30:01.534 INFO:teuthology.orchestra.run.trial051.stdout: Package Architecture Version Repository Size 2026-02-18T06:30:01.534 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:30:01.534 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:30:01.534 INFO:teuthology.orchestra.run.trial051.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:Installing dependencies: 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout: kernel-core x86_64 5.14.0-681.el9 baseos 17 M 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout: kernel-modules x86_64 5.14.0-681.el9 baseos 40 M 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout: kernel-modules-core x86_64 5.14.0-681.el9 baseos 31 M 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:Install 4 Packages 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 88 M 2026-02-18T06:30:01.535 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 135 M 2026-02-18T06:30:01.536 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:30:01.779 INFO:teuthology.orchestra.run.trial051.stdout:(1/4): kernel-5.14.0-681.el9.x86_64.rpm 4.3 MB/s | 803 kB 00:00 2026-02-18T06:30:01.949 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - Extras packages 25 kB/s | 20 kB 00:00 2026-02-18T06:30:02.186 INFO:teuthology.orchestra.run.trial051.stdout:(2/4): kernel-core-5.14.0-681.el9.x86_64.rpm 29 MB/s | 17 MB 00:00 2026-02-18T06:30:02.504 INFO:teuthology.orchestra.run.trial051.stdout:(3/4): kernel-modules-5.14.0-681.el9.x86_64.rpm 44 MB/s | 40 MB 00:00 2026-02-18T06:30:02.610 INFO:teuthology.orchestra.run.trial051.stdout:(4/4): kernel-modules-core-5.14.0-681.el9.x86_6 37 MB/s | 31 MB 00:00 2026-02-18T06:30:02.610 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:30:02.610 INFO:teuthology.orchestra.run.trial051.stdout:Total 82 MB/s | 88 MB 00:01 2026-02-18T06:30:02.805 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:30:02.805 INFO:teuthology.orchestra.run.trial006.stdout:Extra Packages for Enterprise Linux 26 MB/s | 20 MB 00:00 2026-02-18T06:30:03.003 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:30:03.003 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:30:03.349 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:30:03.350 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:30:04.425 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:30:04.704 INFO:teuthology.orchestra.run.trial051.stdout: Installing : kernel-modules-core-5.14.0-681.el9.x86_64 1/4 2026-02-18T06:30:04.723 INFO:teuthology.orchestra.run.trial051.stdout: Installing : kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:05.999 INFO:teuthology.orchestra.run.trial006.stdout:lab-extras 1.5 MB/s | 50 kB 00:00 2026-02-18T06:30:06.500 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:06.530 INFO:teuthology.orchestra.run.trial051.stdout: Installing : kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:07.012 INFO:teuthology.orchestra.run.trial006.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-18T06:30:07.034 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:30:07.035 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:30:07.035 INFO:teuthology.orchestra.run.trial006.stdout: Package Architecture Version Repository Size 2026-02-18T06:30:07.035 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout:Installing dependencies: 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout: kernel-core x86_64 5.14.0-681.el9 baseos 17 M 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout: kernel-modules x86_64 5.14.0-681.el9 baseos 40 M 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout: kernel-modules-core x86_64 5.14.0-681.el9 baseos 31 M 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:30:07.036 INFO:teuthology.orchestra.run.trial006.stdout:Install 4 Packages 2026-02-18T06:30:07.037 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:07.037 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 88 M 2026-02-18T06:30:07.037 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 135 M 2026-02-18T06:30:07.037 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:30:07.983 INFO:teuthology.orchestra.run.trial006.stdout:(1/4): kernel-5.14.0-681.el9.x86_64.rpm 1.2 MB/s | 803 kB 00:00 2026-02-18T06:30:08.543 INFO:teuthology.orchestra.run.trial006.stdout:(2/4): kernel-core-5.14.0-681.el9.x86_64.rpm 14 MB/s | 17 MB 00:01 2026-02-18T06:30:09.076 INFO:teuthology.orchestra.run.trial006.stdout:(3/4): kernel-modules-core-5.14.0-681.el9.x86_6 28 MB/s | 31 MB 00:01 2026-02-18T06:30:09.234 INFO:teuthology.orchestra.run.trial006.stdout:(4/4): kernel-modules-5.14.0-681.el9.x86_64.rpm 21 MB/s | 40 MB 00:01 2026-02-18T06:30:09.234 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:30:09.234 INFO:teuthology.orchestra.run.trial006.stdout:Total 40 MB/s | 88 MB 00:02 2026-02-18T06:30:09.423 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:30:09.623 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:30:09.623 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:30:09.753 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:09.759 INFO:teuthology.orchestra.run.trial051.stdout: Installing : kernel-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:09.970 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:30:09.971 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:30:11.048 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:30:11.328 INFO:teuthology.orchestra.run.trial006.stdout: Installing : kernel-modules-core-5.14.0-681.el9.x86_64 1/4 2026-02-18T06:30:11.347 INFO:teuthology.orchestra.run.trial006.stdout: Installing : kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:12.855 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-modules-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:13.127 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:13.156 INFO:teuthology.orchestra.run.trial006.stdout: Installing : kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:16.184 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:16.191 INFO:teuthology.orchestra.run.trial006.stdout: Installing : kernel-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:19.229 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-modules-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:43.295 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:43.316 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-modules-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:43.379 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:43.379 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/4 2026-02-18T06:30:43.379 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:43.379 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:43.430 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : kernel-modules-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:43.430 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:43.430 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:30:43.430 INFO:teuthology.orchestra.run.trial051.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.431 INFO:teuthology.orchestra.run.trial051.stdout: kernel-core-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.431 INFO:teuthology.orchestra.run.trial051.stdout: kernel-modules-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.431 INFO:teuthology.orchestra.run.trial051.stdout: kernel-modules-core-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.431 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:43.431 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:30:43.490 DEBUG:teuthology.orchestra.run.trial051:> rpm -q kernel | sort -rV | head -n 1 2026-02-18T06:30:43.515 INFO:teuthology.orchestra.run.trial051.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.517 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-18T06:30:43.517 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-645.el9.x86_64 Expected: 5.14.0-681.el9.x86_64 2026-02-18T06:30:43.517 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2026-02-18T06:30:43.517 DEBUG:teuthology.task.kernel:src is distro, skipping download 2026-02-18T06:30:43.517 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2026-02-18T06:30:43.517 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@trial051.front.sepia.ceph.com, path=None, version=distro) 2026-02-18T06:30:43.518 DEBUG:teuthology.orchestra.run.trial051:> rpm -q kernel | sort -rV | head -n 1 2026-02-18T06:30:43.580 INFO:teuthology.orchestra.run.trial051.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.581 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-18T06:30:43.582 DEBUG:teuthology.orchestra.run.trial051:> sudo rpm -qi grub2-tools 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Name : grub2-tools 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Epoch : 1 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Version : 2.06 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Release : 118.el9 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Architecture: x86_64 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Install Date: Thu 04 Dec 2025 11:54:21 PM UTC 2026-02-18T06:30:43.649 INFO:teuthology.orchestra.run.trial051.stdout:Group : Unspecified 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Size : 8393909 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:License : GPLv3+ 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Signature : RSA/SHA256, Wed 05 Nov 2025 09:13:15 AM UTC, Key ID 05b555b38483c65d 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Source RPM : grub2-2.06-118.el9.src.rpm 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Build Date : Tue 04 Nov 2025 03:44:14 PM UTC 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Packager : builder@centos.org 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Vendor : CentOS 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:URL : http://www.gnu.org/software/grub/ 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Summary : Support tools for GRUB. 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout:Description : 2026-02-18T06:30:43.650 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout:customizable bootloader with modular architecture. It supports a rich 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout:variety of kernel formats, file systems, computer architectures and 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout:hardware devices. 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:30:43.651 INFO:teuthology.orchestra.run.trial051.stdout:This subpackage provides tools for support of all platforms. 2026-02-18T06:30:43.652 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2026-02-18T06:30:43.652 INFO:teuthology.task.kernel:Updating grub on trial051 to boot 5.14.0-681.el9.x86_64 2026-02-18T06:30:43.653 DEBUG:teuthology.orchestra.run.trial051:> sudo find /boot/loader/entries -maxdepth 1 -name "*.conf" -type f -print -quit 2026-02-18T06:30:43.715 INFO:teuthology.orchestra.run.trial051.stdout:/boot/loader/entries/b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64.conf 2026-02-18T06:30:43.716 DEBUG:teuthology.orchestra.run.trial051:> sudo command -v grubby && echo yes || echo no 2026-02-18T06:30:43.779 INFO:teuthology.orchestra.run.trial051.stdout:/sbin/grubby 2026-02-18T06:30:43.781 INFO:teuthology.orchestra.run.trial051.stdout:yes 2026-02-18T06:30:43.781 DEBUG:teuthology.orchestra.run.trial051:> sudo find /boot/loader/entries -maxdepth 1 -name "*.conf" -type f | grep -F "5.14.0-681.el9.x86_64" | head -n 1 2026-02-18T06:30:43.844 INFO:teuthology.orchestra.run.trial051.stdout:/boot/loader/entries/b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64.conf 2026-02-18T06:30:43.846 DEBUG:teuthology.orchestra.run.trial051:> sudo grub2-set-default b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64 2026-02-18T06:30:43.918 DEBUG:teuthology.orchestra.run.trial051:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2026-02-18T06:30:44.126 INFO:teuthology.orchestra.run.trial051.stderr:Generating grub configuration file ... 2026-02-18T06:30:44.191 INFO:teuthology.orchestra.run.trial051.stderr:Found linux image: /boot/vmlinuz-5.14.0-681.el9.x86_64 2026-02-18T06:30:44.197 INFO:teuthology.orchestra.run.trial051.stderr:Found initrd image: /boot/initramfs-5.14.0-681.el9.x86_64.img 2026-02-18T06:30:44.446 INFO:teuthology.orchestra.run.trial051.stderr:Found linux image: /boot/vmlinuz-5.14.0-645.el9.x86_64 2026-02-18T06:30:44.452 INFO:teuthology.orchestra.run.trial051.stderr:Found initrd image: /boot/initramfs-5.14.0-645.el9.x86_64.img 2026-02-18T06:30:44.484 INFO:teuthology.orchestra.run.trial051.stderr:Found linux image: /boot/vmlinuz-0-rescue-b0373be789df49cdafedf423703d2566 2026-02-18T06:30:44.490 INFO:teuthology.orchestra.run.trial051.stderr:Found initrd image: /boot/initramfs-0-rescue-b0373be789df49cdafedf423703d2566.img 2026-02-18T06:30:44.514 INFO:teuthology.orchestra.run.trial051.stderr:Found linux image: /boot/vmlinuz-0-rescue-2734456005d743dca7f3dc77c35a2d0d 2026-02-18T06:30:44.520 INFO:teuthology.orchestra.run.trial051.stderr:Found initrd image: /boot/initramfs-0-rescue-2734456005d743dca7f3dc77c35a2d0d.img 2026-02-18T06:30:44.603 INFO:teuthology.orchestra.run.trial051.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T06:30:44.611 INFO:teuthology.orchestra.run.trial051.stderr:done 2026-02-18T06:30:44.613 DEBUG:teuthology.orchestra.run.trial051:> test -d /sys/firmware/efi 2026-02-18T06:30:44.629 DEBUG:teuthology.orchestra.run.trial051:> sudo find /boot/efi/EFI -mindepth 1 -maxdepth 1 -type d || true 2026-02-18T06:30:44.694 INFO:teuthology.orchestra.run.trial051.stdout:/boot/efi/EFI/BOOT 2026-02-18T06:30:44.694 INFO:teuthology.orchestra.run.trial051.stdout:/boot/efi/EFI/centos 2026-02-18T06:30:44.696 DEBUG:teuthology.orchestra.run.trial051:> sudo test -f /boot/efi/EFI/BOOT/grub.cfg && echo yes || echo no 2026-02-18T06:30:44.761 INFO:teuthology.orchestra.run.trial051.stdout:no 2026-02-18T06:30:44.761 DEBUG:teuthology.orchestra.run.trial051:> sudo test -f /boot/efi/EFI/centos/grub.cfg && echo yes || echo no 2026-02-18T06:30:44.825 INFO:teuthology.orchestra.run.trial051.stdout:yes 2026-02-18T06:30:44.825 DEBUG:teuthology.orchestra.run.trial051:> sudo cp -f /boot/grub2/grub.cfg /boot/efi/EFI/centos/grub.cfg 2026-02-18T06:30:44.890 INFO:teuthology.orchestra.remote:Triggering safe hard reboot via SysRq on trial051 2026-02-18T06:30:44.891 DEBUG:teuthology.orchestra.run.trial051:> sudo bash -c ' 2026-02-18T06:30:44.891 DEBUG:teuthology.orchestra.run.trial051:> echo pci > /sys/kernel/reboot/type && echo 1 > /sys/kernel/reboot/force && sync && echo s > /proc/sysrq-trigger && echo u > /proc/sysrq-trigger && sleep 2 && echo b > /proc/sysrq-trigger 2026-02-18T06:30:44.891 DEBUG:teuthology.orchestra.run.trial051:> ' 2026-02-18T06:30:49.550 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:49.572 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-modules-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:49.637 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:49.637 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/4 2026-02-18T06:30:49.637 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : kernel-core-5.14.0-681.el9.x86_64 2/4 2026-02-18T06:30:49.637 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : kernel-modules-5.14.0-681.el9.x86_64 3/4 2026-02-18T06:30:49.688 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : kernel-modules-core-5.14.0-681.el9.x86_64 4/4 2026-02-18T06:30:49.688 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout: kernel-core-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout: kernel-modules-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout: kernel-modules-core-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:49.689 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:30:49.747 DEBUG:teuthology.orchestra.run.trial006:> rpm -q kernel | sort -rV | head -n 1 2026-02-18T06:30:49.771 INFO:teuthology.orchestra.run.trial006.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.773 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-18T06:30:49.773 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-645.el9.x86_64 Expected: 5.14.0-681.el9.x86_64 2026-02-18T06:30:49.773 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2026-02-18T06:30:49.773 DEBUG:teuthology.task.kernel:src is distro, skipping download 2026-02-18T06:30:49.773 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2026-02-18T06:30:49.774 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@trial006.front.sepia.ceph.com, path=None, version=distro) 2026-02-18T06:30:49.774 DEBUG:teuthology.orchestra.run.trial006:> rpm -q kernel | sort -rV | head -n 1 2026-02-18T06:30:49.835 INFO:teuthology.orchestra.run.trial006.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-18T06:30:49.836 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-18T06:30:49.836 DEBUG:teuthology.orchestra.run.trial006:> sudo rpm -qi grub2-tools 2026-02-18T06:30:49.903 INFO:teuthology.orchestra.run.trial006.stdout:Name : grub2-tools 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Epoch : 1 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Version : 2.06 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Release : 118.el9 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Architecture: x86_64 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Install Date: Thu 04 Dec 2025 11:54:21 PM UTC 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Group : Unspecified 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Size : 8393909 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:License : GPLv3+ 2026-02-18T06:30:49.904 INFO:teuthology.orchestra.run.trial006.stdout:Signature : RSA/SHA256, Wed 05 Nov 2025 09:13:15 AM UTC, Key ID 05b555b38483c65d 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Source RPM : grub2-2.06-118.el9.src.rpm 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Build Date : Tue 04 Nov 2025 03:44:14 PM UTC 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Packager : builder@centos.org 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Vendor : CentOS 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:URL : http://www.gnu.org/software/grub/ 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Summary : Support tools for GRUB. 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:Description : 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:customizable bootloader with modular architecture. It supports a rich 2026-02-18T06:30:49.905 INFO:teuthology.orchestra.run.trial006.stdout:variety of kernel formats, file systems, computer architectures and 2026-02-18T06:30:49.906 INFO:teuthology.orchestra.run.trial006.stdout:hardware devices. 2026-02-18T06:30:49.906 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:30:49.906 INFO:teuthology.orchestra.run.trial006.stdout:This subpackage provides tools for support of all platforms. 2026-02-18T06:30:49.907 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2026-02-18T06:30:49.907 INFO:teuthology.task.kernel:Updating grub on trial006 to boot 5.14.0-681.el9.x86_64 2026-02-18T06:30:49.907 DEBUG:teuthology.orchestra.run.trial006:> sudo find /boot/loader/entries -maxdepth 1 -name "*.conf" -type f -print -quit 2026-02-18T06:30:49.972 INFO:teuthology.orchestra.run.trial006.stdout:/boot/loader/entries/b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64.conf 2026-02-18T06:30:49.973 DEBUG:teuthology.orchestra.run.trial006:> sudo command -v grubby && echo yes || echo no 2026-02-18T06:30:50.036 INFO:teuthology.orchestra.run.trial006.stdout:/sbin/grubby 2026-02-18T06:30:50.037 INFO:teuthology.orchestra.run.trial006.stdout:yes 2026-02-18T06:30:50.038 DEBUG:teuthology.orchestra.run.trial006:> sudo find /boot/loader/entries -maxdepth 1 -name "*.conf" -type f | grep -F "5.14.0-681.el9.x86_64" | head -n 1 2026-02-18T06:30:50.101 INFO:teuthology.orchestra.run.trial006.stdout:/boot/loader/entries/b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64.conf 2026-02-18T06:30:50.103 DEBUG:teuthology.orchestra.run.trial006:> sudo grub2-set-default b0373be789df49cdafedf423703d2566-5.14.0-681.el9.x86_64 2026-02-18T06:30:50.176 DEBUG:teuthology.orchestra.run.trial006:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2026-02-18T06:30:50.384 INFO:teuthology.orchestra.run.trial006.stderr:Generating grub configuration file ... 2026-02-18T06:30:50.448 INFO:teuthology.orchestra.run.trial006.stderr:Found linux image: /boot/vmlinuz-5.14.0-681.el9.x86_64 2026-02-18T06:30:50.455 INFO:teuthology.orchestra.run.trial006.stderr:Found initrd image: /boot/initramfs-5.14.0-681.el9.x86_64.img 2026-02-18T06:30:50.703 INFO:teuthology.orchestra.run.trial006.stderr:Found linux image: /boot/vmlinuz-5.14.0-645.el9.x86_64 2026-02-18T06:30:50.709 INFO:teuthology.orchestra.run.trial006.stderr:Found initrd image: /boot/initramfs-5.14.0-645.el9.x86_64.img 2026-02-18T06:30:50.740 INFO:teuthology.orchestra.run.trial006.stderr:Found linux image: /boot/vmlinuz-0-rescue-b0373be789df49cdafedf423703d2566 2026-02-18T06:30:50.747 INFO:teuthology.orchestra.run.trial006.stderr:Found initrd image: /boot/initramfs-0-rescue-b0373be789df49cdafedf423703d2566.img 2026-02-18T06:30:50.770 INFO:teuthology.orchestra.run.trial006.stderr:Found linux image: /boot/vmlinuz-0-rescue-2734456005d743dca7f3dc77c35a2d0d 2026-02-18T06:30:50.776 INFO:teuthology.orchestra.run.trial006.stderr:Found initrd image: /boot/initramfs-0-rescue-2734456005d743dca7f3dc77c35a2d0d.img 2026-02-18T06:30:50.859 INFO:teuthology.orchestra.run.trial006.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2026-02-18T06:30:50.867 INFO:teuthology.orchestra.run.trial006.stderr:done 2026-02-18T06:30:50.869 DEBUG:teuthology.orchestra.run.trial006:> test -d /sys/firmware/efi 2026-02-18T06:30:50.885 DEBUG:teuthology.orchestra.run.trial006:> sudo find /boot/efi/EFI -mindepth 1 -maxdepth 1 -type d || true 2026-02-18T06:30:50.950 INFO:teuthology.orchestra.run.trial006.stdout:/boot/efi/EFI/BOOT 2026-02-18T06:30:50.950 INFO:teuthology.orchestra.run.trial006.stdout:/boot/efi/EFI/centos 2026-02-18T06:30:50.952 DEBUG:teuthology.orchestra.run.trial006:> sudo test -f /boot/efi/EFI/BOOT/grub.cfg && echo yes || echo no 2026-02-18T06:30:51.019 INFO:teuthology.orchestra.run.trial006.stdout:no 2026-02-18T06:30:51.020 DEBUG:teuthology.orchestra.run.trial006:> sudo test -f /boot/efi/EFI/centos/grub.cfg && echo yes || echo no 2026-02-18T06:30:51.083 INFO:teuthology.orchestra.run.trial006.stdout:yes 2026-02-18T06:30:51.084 DEBUG:teuthology.orchestra.run.trial006:> sudo cp -f /boot/grub2/grub.cfg /boot/efi/EFI/centos/grub.cfg 2026-02-18T06:30:51.147 INFO:teuthology.orchestra.remote:Triggering safe hard reboot via SysRq on trial006 2026-02-18T06:30:51.148 DEBUG:teuthology.orchestra.run.trial006:> sudo bash -c ' 2026-02-18T06:30:51.148 DEBUG:teuthology.orchestra.run.trial006:> echo pci > /sys/kernel/reboot/type && echo 1 > /sys/kernel/reboot/force && sync && echo s > /proc/sysrq-trigger && echo u > /proc/sysrq-trigger && sleep 2 && echo b > /proc/sysrq-trigger 2026-02-18T06:30:51.148 DEBUG:teuthology.orchestra.run.trial006:> ' 2026-02-18T06:31:14.932 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2026-02-18T06:31:14.933 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial051.front.sepia.ceph.com' 2026-02-18T06:31:14.934 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial051.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:31:21.190 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2026-02-18T06:31:21.191 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial006.front.sepia.ceph.com' 2026-02-18T06:31:21.192 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:31:29.216 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 10.20.193.51 2026-02-18T06:31:35.488 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 10.20.193.6 2026-02-18T06:31:38.217 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial051.front.sepia.ceph.com' 2026-02-18T06:31:38.222 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial051.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:31:44.494 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial006.front.sepia.ceph.com' 2026-02-18T06:31:44.495 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:32:14.127 DEBUG:teuthology.orchestra.run.trial051:> true 2026-02-18T06:32:14.292 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial051.front.sepia.ceph.com' 2026-02-18T06:32:14.292 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.14.0-681.el9.x86_64"... 2026-02-18T06:32:14.293 DEBUG:teuthology.orchestra.run.trial051:> uname -r 2026-02-18T06:32:14.350 INFO:teuthology.orchestra.run.trial051.stdout:5.14.0-681.el9.x86_64 2026-02-18T06:32:14.351 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-681.el9.x86_64 vs 5.14.0-681.el9.x86_64 2026-02-18T06:32:14.351 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2026-02-18T06:32:14.351 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2026-02-18T06:32:15.352 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-18T06:32:15.352 DEBUG:teuthology.orchestra.run.trial051:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-18T06:32:15.382 INFO:teuthology.orchestra.run.trial051.stdout:ttyS1 2026-02-18T06:32:15.407 DEBUG:teuthology.parallel:result is None 2026-02-18T06:32:20.741 DEBUG:teuthology.orchestra.run.trial006:> true 2026-02-18T06:32:20.896 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial006.front.sepia.ceph.com' 2026-02-18T06:32:20.896 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.14.0-681.el9.x86_64"... 2026-02-18T06:32:20.896 DEBUG:teuthology.orchestra.run.trial006:> uname -r 2026-02-18T06:32:20.952 INFO:teuthology.orchestra.run.trial006.stdout:5.14.0-681.el9.x86_64 2026-02-18T06:32:20.952 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-681.el9.x86_64 vs 5.14.0-681.el9.x86_64 2026-02-18T06:32:20.953 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2026-02-18T06:32:20.953 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2026-02-18T06:32:21.954 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-18T06:32:21.954 DEBUG:teuthology.orchestra.run.trial006:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-18T06:32:21.983 INFO:teuthology.orchestra.run.trial006.stdout:ttyS1 2026-02-18T06:32:22.004 DEBUG:teuthology.parallel:result is None 2026-02-18T06:32:22.004 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-18T06:32:22.019 INFO:teuthology.task.internal:Creating test directory... 2026-02-18T06:32:22.019 DEBUG:teuthology.orchestra.run.trial006:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-18T06:32:22.047 DEBUG:teuthology.orchestra.run.trial051:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-18T06:32:22.065 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-18T06:32:22.076 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-18T06:32:22.088 INFO:teuthology.task.internal:Creating archive directory... 2026-02-18T06:32:22.088 DEBUG:teuthology.orchestra.run.trial006:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-18T06:32:22.104 DEBUG:teuthology.orchestra.run.trial051:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-18T06:32:22.132 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-18T06:32:22.143 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-18T06:32:22.144 DEBUG:teuthology.orchestra.run.trial006:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-18T06:32:22.162 DEBUG:teuthology.orchestra.run.trial051:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-18T06:32:22.184 INFO:teuthology.orchestra.run.trial006.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T06:32:22.186 INFO:teuthology.orchestra.run.trial051.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T06:32:22.192 INFO:teuthology.orchestra.run.trial006.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T06:32:22.195 INFO:teuthology.orchestra.run.trial051.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-18T06:32:22.197 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-18T06:32:22.208 INFO:teuthology.task.internal:Configuring sudo... 2026-02-18T06:32:22.209 DEBUG:teuthology.orchestra.run.trial006:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-18T06:32:22.236 DEBUG:teuthology.orchestra.run.trial051:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-18T06:32:22.262 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-18T06:32:22.282 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-18T06:32:22.282 DEBUG:teuthology.orchestra.run.trial006:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-18T06:32:22.301 DEBUG:teuthology.orchestra.run.trial051:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-18T06:32:22.318 DEBUG:teuthology.orchestra.run.trial006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T06:32:22.477 DEBUG:teuthology.orchestra.run.trial006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T06:32:22.503 DEBUG:teuthology.orchestra.run.trial006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T06:32:22.653 DEBUG:teuthology.orchestra.run.trial006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T06:32:22.676 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:32:22.676 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-18T06:32:22.742 DEBUG:teuthology.orchestra.run.trial051:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T06:32:22.850 DEBUG:teuthology.orchestra.run.trial051:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T06:32:22.875 DEBUG:teuthology.orchestra.run.trial051:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T06:32:23.002 DEBUG:teuthology.orchestra.run.trial051:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T06:32:23.027 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:32:23.027 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-18T06:32:23.091 DEBUG:teuthology.orchestra.run.trial006:> sudo service rsyslog restart 2026-02-18T06:32:23.093 DEBUG:teuthology.orchestra.run.trial051:> sudo service rsyslog restart 2026-02-18T06:32:23.120 INFO:teuthology.orchestra.run.trial006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-18T06:32:23.158 INFO:teuthology.orchestra.run.trial051.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-18T06:32:23.677 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-18T06:32:23.690 INFO:teuthology.task.internal:Starting timer... 2026-02-18T06:32:23.691 INFO:teuthology.run_tasks:Running task pcp... 2026-02-18T06:32:23.713 INFO:teuthology.run_tasks:Running task selinux... 2026-02-18T06:32:23.739 DEBUG:teuthology.orchestra.run.trial006:> sudo service auditd rotate 2026-02-18T06:32:23.769 INFO:teuthology.orchestra.run.trial006.stdout:Rotating logs: 2026-02-18T06:32:23.771 DEBUG:teuthology.orchestra.run.trial051:> sudo service auditd rotate 2026-02-18T06:32:23.800 INFO:teuthology.orchestra.run.trial051.stdout:Rotating logs: 2026-02-18T06:32:23.802 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-18T06:32:23.802 DEBUG:teuthology.orchestra.run.trial006:> /usr/sbin/getenforce 2026-02-18T06:32:23.828 INFO:teuthology.orchestra.run.trial006.stdout:Permissive 2026-02-18T06:32:23.829 DEBUG:teuthology.orchestra.run.trial051:> /usr/sbin/getenforce 2026-02-18T06:32:23.858 INFO:teuthology.orchestra.run.trial051.stdout:Permissive 2026-02-18T06:32:23.859 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial006.front.sepia.ceph.com': 'permissive', 'ubuntu@trial051.front.sepia.ceph.com': 'permissive'} 2026-02-18T06:32:23.859 DEBUG:teuthology.orchestra.run.trial006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-18T06:32:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T06:32:23.893 DEBUG:teuthology.orchestra.run.trial051:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-18T06:32:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T06:32:23.923 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-18T06:32:23.923 DEBUG:teuthology.orchestra.run.trial006:> sudo /usr/sbin/setenforce permissive 2026-02-18T06:32:23.956 DEBUG:teuthology.orchestra.run.trial051:> sudo /usr/sbin/setenforce permissive 2026-02-18T06:32:23.986 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-18T06:32:24.002 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-18T06:32:24.014 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-18T06:32:24.016 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial006.front.sepia.ceph.com,trial051.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-18T06:33:33.904 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial006.front.sepia.ceph.com'), Remote(name='ubuntu@trial051.front.sepia.ceph.com')] 2026-02-18T06:33:33.905 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial006.front.sepia.ceph.com' 2026-02-18T06:33:33.906 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:33:34.016 DEBUG:teuthology.orchestra.run.trial006:> true 2026-02-18T06:33:34.094 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial006.front.sepia.ceph.com' 2026-02-18T06:33:34.095 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial051.front.sepia.ceph.com' 2026-02-18T06:33:34.095 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial051.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-18T06:33:34.189 DEBUG:teuthology.orchestra.run.trial051:> true 2026-02-18T06:33:34.267 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial051.front.sepia.ceph.com' 2026-02-18T06:33:34.267 INFO:teuthology.run_tasks:Running task clock... 2026-02-18T06:33:34.283 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-18T06:33:34.284 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-18T06:33:34.285 DEBUG:teuthology.orchestra.run.trial006:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T06:33:34.287 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-18T06:33:34.288 DEBUG:teuthology.orchestra.run.trial051:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T06:33:34.313 INFO:teuthology.orchestra.run.trial006.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-18T06:33:34.326 INFO:teuthology.orchestra.run.trial006.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-18T06:33:34.333 INFO:teuthology.orchestra.run.trial051.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-18T06:33:34.345 INFO:teuthology.orchestra.run.trial051.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-18T06:33:34.347 INFO:teuthology.orchestra.run.trial006.stderr:sudo: ntpd: command not found 2026-02-18T06:33:34.356 INFO:teuthology.orchestra.run.trial006.stdout:506 Cannot talk to daemon 2026-02-18T06:33:34.366 INFO:teuthology.orchestra.run.trial051.stderr:sudo: ntpd: command not found 2026-02-18T06:33:34.368 INFO:teuthology.orchestra.run.trial006.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-18T06:33:34.376 INFO:teuthology.orchestra.run.trial051.stdout:506 Cannot talk to daemon 2026-02-18T06:33:34.379 INFO:teuthology.orchestra.run.trial006.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-18T06:33:34.388 INFO:teuthology.orchestra.run.trial051.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-18T06:33:34.399 INFO:teuthology.orchestra.run.trial051.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-18T06:33:34.429 INFO:teuthology.orchestra.run.trial006.stderr:bash: line 1: ntpq: command not found 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:=============================================================================== 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:^? 23.186.168.131 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:^? lax1.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:^? 172-104-28-175.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.445 INFO:teuthology.orchestra.run.trial006.stdout:^? rn-02.koehn.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.446 INFO:teuthology.orchestra.run.trial051.stderr:bash: line 1: ntpq: command not found 2026-02-18T06:33:34.466 INFO:teuthology.orchestra.run.trial051.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-18T06:33:34.466 INFO:teuthology.orchestra.run.trial051.stdout:=============================================================================== 2026-02-18T06:33:34.466 INFO:teuthology.orchestra.run.trial051.stdout:^? time-usw1.crimpac.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.466 INFO:teuthology.orchestra.run.trial051.stdout:^? dns-e.ns4v.icu 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.467 INFO:teuthology.orchestra.run.trial051.stdout:^? arm1.maxhost.io 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.467 INFO:teuthology.orchestra.run.trial051.stdout:^? hou1.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-18T06:33:34.468 INFO:teuthology.run_tasks:Running task install... 2026-02-18T06:33:34.482 DEBUG:teuthology.task.install:project ceph 2026-02-18T06:33:34.482 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}} 2026-02-18T06:33:34.482 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'} 2026-02-18T06:33:34.482 INFO:teuthology.task.install:Using flavor: default 2026-02-18T06:33:34.491 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-18T06:33:34.491 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-18T06:33:34.491 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'tag': None, 'wait_for_package': False} 2026-02-18T06:33:34.492 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:33:34.495 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'tag': None, 'wait_for_package': False} 2026-02-18T06:33:34.495 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:33:34.694 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/ 2026-02-18T06:33:34.694 INFO:teuthology.task.install.rpm:Package version is 20.2.0-670.gadc19233 2026-02-18T06:33:34.703 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/ 2026-02-18T06:33:34.703 INFO:teuthology.task.install.rpm:Package version is 20.2.0-670.gadc19233 2026-02-18T06:33:34.846 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-18T06:33:34.846 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:33:34.846 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-18T06:33:34.859 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/20.2.1_2.17.26-baseline_2/adc1923395f2bcd6ae89b5d331d278364d1848ed/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-18T06:33:34.859 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:33:34.859 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-18T06:33:34.871 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-18T06:33:34.871 DEBUG:teuthology.orchestra.run.trial006:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/adc1923395f2bcd6ae89b5d331d278364d1848ed/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-18T06:33:34.884 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-18T06:33:34.884 DEBUG:teuthology.orchestra.run.trial051:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/adc1923395f2bcd6ae89b5d331d278364d1848ed/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-18T06:33:34.935 DEBUG:teuthology.orchestra.run.trial006:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-18T06:33:34.948 DEBUG:teuthology.orchestra.run.trial051:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-18T06:33:35.008 DEBUG:teuthology.orchestra.run.trial006:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-18T06:33:35.021 DEBUG:teuthology.orchestra.run.trial051:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-18T06:33:35.072 INFO:teuthology.orchestra.run.trial006.stdout:check_obsoletes = 1 2026-02-18T06:33:35.073 DEBUG:teuthology.orchestra.run.trial006:> sudo yum clean all 2026-02-18T06:33:35.084 INFO:teuthology.orchestra.run.trial051.stdout:check_obsoletes = 1 2026-02-18T06:33:35.086 DEBUG:teuthology.orchestra.run.trial051:> sudo yum clean all 2026-02-18T06:33:35.222 INFO:teuthology.orchestra.run.trial006.stdout:41 files removed 2026-02-18T06:33:35.232 INFO:teuthology.orchestra.run.trial051.stdout:41 files removed 2026-02-18T06:33:35.236 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-radosgw 2026-02-18T06:33:35.246 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-radosgw 2026-02-18T06:33:35.504 INFO:teuthology.orchestra.run.trial006.stdout:ceph packages for x86_64 721 kB/s | 89 kB 00:00 2026-02-18T06:33:35.515 INFO:teuthology.orchestra.run.trial051.stdout:ceph packages for x86_64 706 kB/s | 89 kB 00:00 2026-02-18T06:33:35.646 INFO:teuthology.orchestra.run.trial006.stdout:ceph noarch packages 160 kB/s | 19 kB 00:00 2026-02-18T06:33:35.653 INFO:teuthology.orchestra.run.trial051.stdout:ceph noarch packages 167 kB/s | 19 kB 00:00 2026-02-18T06:33:35.770 INFO:teuthology.orchestra.run.trial006.stdout:ceph source packages 19 kB/s | 1.9 kB 00:00 2026-02-18T06:33:35.778 INFO:teuthology.orchestra.run.trial051.stdout:ceph source packages 19 kB/s | 1.9 kB 00:00 2026-02-18T06:33:36.479 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-18T06:33:36.489 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-18T06:33:39.641 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - AppStream 10 MB/s | 27 MB 00:02 2026-02-18T06:33:40.224 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - AppStream 8.3 MB/s | 27 MB 00:03 2026-02-18T06:33:42.962 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - CRB 8.4 MB/s | 7.8 MB 00:00 2026-02-18T06:33:43.502 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - CRB 8.7 MB/s | 7.8 MB 00:00 2026-02-18T06:33:43.872 INFO:teuthology.orchestra.run.trial051.stdout:CentOS Stream 9 - Extras packages 108 kB/s | 20 kB 00:00 2026-02-18T06:33:44.548 INFO:teuthology.orchestra.run.trial051.stdout:Extra Packages for Enterprise Linux 33 MB/s | 20 MB 00:00 2026-02-18T06:33:44.568 INFO:teuthology.orchestra.run.trial006.stdout:CentOS Stream 9 - Extras packages 59 kB/s | 20 kB 00:00 2026-02-18T06:33:44.992 INFO:teuthology.orchestra.run.trial006.stdout:Extra Packages for Enterprise Linux 56 MB/s | 20 MB 00:00 2026-02-18T06:33:47.705 INFO:teuthology.orchestra.run.trial051.stdout:lab-extras 2.5 MB/s | 50 kB 00:00 2026-02-18T06:33:48.176 INFO:teuthology.orchestra.run.trial006.stdout:lab-extras 2.5 MB/s | 50 kB 00:00 2026-02-18T06:33:48.933 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repository Size 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout: ceph-radosgw x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 M 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout:Upgrading: 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout: librados2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 3.5 M 2026-02-18T06:33:48.935 INFO:teuthology.orchestra.run.trial051.stdout: librbd1 x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.8 M 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout:Installing dependencies: 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: ceph-base x86_64 2:20.2.0-670.gadc19233.el9 ceph 5.9 M 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: ceph-common x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 M 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: ceph-selinux x86_64 2:20.2.0-670.gadc19233.el9 ceph 25 k 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 869 k 2026-02-18T06:33:48.936 INFO:teuthology.orchestra.run.trial051.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: libradosstriper1 x86_64 2:20.2.0-670.gadc19233.el9 ceph 250 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: librgw2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 6.4 M 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-18T06:33:48.937 INFO:teuthology.orchestra.run.trial051.stdout: python3-ceph-argparse x86_64 2:20.2.0-670.gadc19233.el9 ceph 45 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: python3-ceph-common x86_64 2:20.2.0-670.gadc19233.el9 ceph 175 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: python3-cephfs x86_64 2:20.2.0-670.gadc19233.el9 ceph 162 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: python3-rados x86_64 2:20.2.0-670.gadc19233.el9 ceph 323 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: python3-rbd x86_64 2:20.2.0-670.gadc19233.el9 ceph 305 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: python3-rgw x86_64 2:20.2.0-670.gadc19233.el9 ceph 99 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-18T06:33:48.938 INFO:teuthology.orchestra.run.trial051.stdout:Installing weak dependencies: 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-daemon x86_64 2:20.2.0-670.gadc19233.el9 ceph 35 k 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-proxy2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 k 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: nvme-cli x86_64 2.13-1.el9 baseos 1.1 M 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout:Install 35 Packages 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout:Upgrade 2 Packages 2026-02-18T06:33:48.939 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:48.940 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 79 M 2026-02-18T06:33:48.940 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:33:49.400 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:33:49.401 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:33:49.401 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repository Size 2026-02-18T06:33:49.401 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: ceph-radosgw x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 M 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout:Upgrading: 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: librados2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 3.5 M 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: librbd1 x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.8 M 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout:Installing dependencies: 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: ceph-base x86_64 2:20.2.0-670.gadc19233.el9 ceph 5.9 M 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: ceph-common x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 M 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: ceph-selinux x86_64 2:20.2.0-670.gadc19233.el9 ceph 25 k 2026-02-18T06:33:49.402 INFO:teuthology.orchestra.run.trial006.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 869 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: libradosstriper1 x86_64 2:20.2.0-670.gadc19233.el9 ceph 250 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-18T06:33:49.403 INFO:teuthology.orchestra.run.trial006.stdout: librgw2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 6.4 M 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-ceph-argparse x86_64 2:20.2.0-670.gadc19233.el9 ceph 45 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-ceph-common x86_64 2:20.2.0-670.gadc19233.el9 ceph 175 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-cephfs x86_64 2:20.2.0-670.gadc19233.el9 ceph 162 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-rados x86_64 2:20.2.0-670.gadc19233.el9 ceph 323 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-rbd x86_64 2:20.2.0-670.gadc19233.el9 ceph 305 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: python3-rgw x86_64 2:20.2.0-670.gadc19233.el9 ceph 99 k 2026-02-18T06:33:49.404 INFO:teuthology.orchestra.run.trial006.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout:Installing weak dependencies: 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-daemon x86_64 2:20.2.0-670.gadc19233.el9 ceph 35 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-proxy2 x86_64 2:20.2.0-670.gadc19233.el9 ceph 24 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: nvme-cli x86_64 2.13-1.el9 baseos 1.1 M 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:33:49.405 INFO:teuthology.orchestra.run.trial006.stdout:Install 35 Packages 2026-02-18T06:33:49.406 INFO:teuthology.orchestra.run.trial006.stdout:Upgrade 2 Packages 2026-02-18T06:33:49.406 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:33:49.406 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 79 M 2026-02-18T06:33:49.406 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:33:49.725 INFO:teuthology.orchestra.run.trial051.stdout:(1/37): ceph-base-20.2.0-670.gadc19233.el9.x86_ 17 MB/s | 5.9 MB 00:00 2026-02-18T06:33:49.761 INFO:teuthology.orchestra.run.trial051.stdout:(2/37): ceph-selinux-20.2.0-670.gadc19233.el9.x 691 kB/s | 25 kB 00:00 2026-02-18T06:33:49.801 INFO:teuthology.orchestra.run.trial051.stdout:(3/37): libcephfs-daemon-20.2.0-670.gadc19233.e 879 kB/s | 35 kB 00:00 2026-02-18T06:33:49.844 INFO:teuthology.orchestra.run.trial051.stdout:(4/37): libcephfs-proxy2-20.2.0-670.gadc19233.e 561 kB/s | 24 kB 00:00 2026-02-18T06:33:49.905 INFO:teuthology.orchestra.run.trial051.stdout:(5/37): libcephfs2-20.2.0-670.gadc19233.el9.x86 14 MB/s | 869 kB 00:00 2026-02-18T06:33:49.956 INFO:teuthology.orchestra.run.trial051.stdout:(6/37): libradosstriper1-20.2.0-670.gadc19233.e 4.8 MB/s | 250 kB 00:00 2026-02-18T06:33:50.322 INFO:teuthology.orchestra.run.trial051.stdout:(7/37): librgw2-20.2.0-670.gadc19233.el9.x86_64 17 MB/s | 6.4 MB 00:00 2026-02-18T06:33:50.361 INFO:teuthology.orchestra.run.trial051.stdout:(8/37): python3-ceph-argparse-20.2.0-670.gadc19 1.1 MB/s | 45 kB 00:00 2026-02-18T06:33:50.406 INFO:teuthology.orchestra.run.trial051.stdout:(9/37): python3-ceph-common-20.2.0-670.gadc1923 3.8 MB/s | 175 kB 00:00 2026-02-18T06:33:50.455 INFO:teuthology.orchestra.run.trial051.stdout:(10/37): python3-cephfs-20.2.0-670.gadc19233.el 3.2 MB/s | 162 kB 00:00 2026-02-18T06:33:50.515 INFO:teuthology.orchestra.run.trial051.stdout:(11/37): python3-rados-20.2.0-670.gadc19233.el9 5.3 MB/s | 323 kB 00:00 2026-02-18T06:33:50.594 INFO:teuthology.orchestra.run.trial051.stdout:(12/37): ceph-common-20.2.0-670.gadc19233.el9.x 20 MB/s | 24 MB 00:01 2026-02-18T06:33:50.595 INFO:teuthology.orchestra.run.trial051.stdout:(13/37): python3-rbd-20.2.0-670.gadc19233.el9.x 3.7 MB/s | 305 kB 00:00 2026-02-18T06:33:50.629 INFO:teuthology.orchestra.run.trial051.stdout:(14/37): ceph-radosgw-20.2.0-670.gadc19233.el9. 19 MB/s | 24 MB 00:01 2026-02-18T06:33:50.630 INFO:teuthology.orchestra.run.trial051.stdout:(15/37): python3-rgw-20.2.0-670.gadc19233.el9.x 2.7 MB/s | 99 kB 00:00 2026-02-18T06:33:50.648 INFO:teuthology.orchestra.run.trial051.stdout:(16/37): mailcap-2.1.49-5.el9.noarch.rpm 627 kB/s | 33 kB 00:00 2026-02-18T06:33:50.665 INFO:teuthology.orchestra.run.trial051.stdout:(17/37): pciutils-3.7.0-7.el9.x86_64.rpm 2.6 MB/s | 93 kB 00:00 2026-02-18T06:33:50.665 INFO:teuthology.orchestra.run.trial051.stdout:(18/37): boost-program-options-1.75.0-13.el9.x8 5.9 MB/s | 104 kB 00:00 2026-02-18T06:33:50.674 INFO:teuthology.orchestra.run.trial051.stdout:(19/37): libnbd-1.20.3-4.el9.x86_64.rpm 18 MB/s | 164 kB 00:00 2026-02-18T06:33:50.678 INFO:teuthology.orchestra.run.trial051.stdout:(20/37): librabbitmq-0.11.0-7.el9.x86_64.rpm 10 MB/s | 45 kB 00:00 2026-02-18T06:33:50.683 INFO:teuthology.orchestra.run.trial051.stdout:(21/37): libpmemobj-1.12.1-1.el9.x86_64.rpm 8.9 MB/s | 160 kB 00:00 2026-02-18T06:33:50.689 INFO:teuthology.orchestra.run.trial051.stdout:(22/37): librdkafka-1.6.1-102.el9.x86_64.rpm 60 MB/s | 662 kB 00:00 2026-02-18T06:33:50.693 INFO:teuthology.orchestra.run.trial051.stdout:(23/37): nvme-cli-2.13-1.el9.x86_64.rpm 17 MB/s | 1.1 MB 00:00 2026-02-18T06:33:50.695 INFO:teuthology.orchestra.run.trial051.stdout:(24/37): qatlib-25.08.0-2.el9.x86_64.rpm 39 MB/s | 240 kB 00:00 2026-02-18T06:33:50.696 INFO:teuthology.orchestra.run.trial051.stdout:(25/37): lttng-ust-2.12.0-6.el9.x86_64.rpm 23 MB/s | 292 kB 00:00 2026-02-18T06:33:50.700 INFO:teuthology.orchestra.run.trial051.stdout:(26/37): qatzip-libs-1.3.1-1.el9.x86_64.rpm 14 MB/s | 66 kB 00:00 2026-02-18T06:33:50.704 INFO:teuthology.orchestra.run.trial051.stdout:(27/37): qatlib-service-25.08.0-2.el9.x86_64.rp 3.2 MB/s | 37 kB 00:00 2026-02-18T06:33:50.718 INFO:teuthology.orchestra.run.trial051.stdout:(28/37): libarrow-doc-9.0.0-15.el9.noarch.rpm 1.8 MB/s | 25 kB 00:00 2026-02-18T06:33:50.727 INFO:teuthology.orchestra.run.trial051.stdout:(29/37): liboath-2.6.12-1.el9.x86_64.rpm 5.6 MB/s | 49 kB 00:00 2026-02-18T06:33:50.733 INFO:teuthology.orchestra.run.trial051.stdout:(30/37): gperftools-libs-2.9.1-3.el9.x86_64.rpm 8.3 MB/s | 308 kB 00:00 2026-02-18T06:33:50.733 INFO:teuthology.orchestra.run.trial051.stdout:(31/37): libunwind-1.6.2-1.el9.x86_64.rpm 11 MB/s | 67 kB 00:00 2026-02-18T06:33:50.745 INFO:teuthology.orchestra.run.trial051.stdout:(32/37): re2-20211101-20.el9.x86_64.rpm 16 MB/s | 191 kB 00:00 2026-02-18T06:33:50.747 INFO:teuthology.orchestra.run.trial051.stdout:(33/37): parquet-libs-9.0.0-15.el9.x86_64.rpm 58 MB/s | 838 kB 00:00 2026-02-18T06:33:50.753 INFO:teuthology.orchestra.run.trial051.stdout:(34/37): libarrow-9.0.0-15.el9.x86_64.rpm 83 MB/s | 4.4 MB 00:00 2026-02-18T06:33:50.777 INFO:teuthology.orchestra.run.trial006.stdout:(1/37): ceph-base-20.2.0-670.gadc19233.el9.x86_ 7.3 MB/s | 5.9 MB 00:00 2026-02-18T06:33:50.781 INFO:teuthology.orchestra.run.trial051.stdout:(35/37): thrift-0.15.0-4.el9.x86_64.rpm 44 MB/s | 1.6 MB 00:00 2026-02-18T06:33:50.832 INFO:teuthology.orchestra.run.trial006.stdout:(2/37): ceph-selinux-20.2.0-670.gadc19233.el9.x 460 kB/s | 25 kB 00:00 2026-02-18T06:33:50.888 INFO:teuthology.orchestra.run.trial006.stdout:(3/37): libcephfs-daemon-20.2.0-670.gadc19233.e 629 kB/s | 35 kB 00:00 2026-02-18T06:33:50.924 INFO:teuthology.orchestra.run.trial051.stdout:(36/37): librbd1-20.2.0-670.gadc19233.el9.x86_6 17 MB/s | 2.8 MB 00:00 2026-02-18T06:33:50.937 INFO:teuthology.orchestra.run.trial006.stdout:(4/37): libcephfs-proxy2-20.2.0-670.gadc19233.e 493 kB/s | 24 kB 00:00 2026-02-18T06:33:50.942 INFO:teuthology.orchestra.run.trial051.stdout:(37/37): librados2-20.2.0-670.gadc19233.el9.x86 18 MB/s | 3.5 MB 00:00 2026-02-18T06:33:50.944 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:33:50.944 INFO:teuthology.orchestra.run.trial051.stdout:Total 39 MB/s | 79 MB 00:02 2026-02-18T06:33:50.995 INFO:teuthology.orchestra.run.trial006.stdout:(5/37): libcephfs2-20.2.0-670.gadc19233.el9.x86 15 MB/s | 869 kB 00:00 2026-02-18T06:33:51.023 INFO:teuthology.orchestra.run.trial006.stdout:(6/37): libradosstriper1-20.2.0-670.gadc19233.e 8.6 MB/s | 250 kB 00:00 2026-02-18T06:33:51.051 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:33:51.070 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:33:51.070 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:33:51.369 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:33:51.369 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:33:51.408 INFO:teuthology.orchestra.run.trial006.stdout:(7/37): librgw2-20.2.0-670.gadc19233.el9.x86_64 17 MB/s | 6.4 MB 00:00 2026-02-18T06:33:51.452 INFO:teuthology.orchestra.run.trial006.stdout:(8/37): python3-ceph-argparse-20.2.0-670.gadc19 1.0 MB/s | 45 kB 00:00 2026-02-18T06:33:51.500 INFO:teuthology.orchestra.run.trial006.stdout:(9/37): python3-ceph-common-20.2.0-670.gadc1923 3.6 MB/s | 175 kB 00:00 2026-02-18T06:33:51.550 INFO:teuthology.orchestra.run.trial006.stdout:(10/37): python3-cephfs-20.2.0-670.gadc19233.el 3.2 MB/s | 162 kB 00:00 2026-02-18T06:33:51.610 INFO:teuthology.orchestra.run.trial006.stdout:(11/37): ceph-common-20.2.0-670.gadc19233.el9.x 14 MB/s | 24 MB 00:01 2026-02-18T06:33:51.611 INFO:teuthology.orchestra.run.trial006.stdout:(12/37): python3-rados-20.2.0-670.gadc19233.el9 5.1 MB/s | 323 kB 00:00 2026-02-18T06:33:51.634 INFO:teuthology.orchestra.run.trial006.stdout:(13/37): python3-rbd-20.2.0-670.gadc19233.el9.x 12 MB/s | 305 kB 00:00 2026-02-18T06:33:51.645 INFO:teuthology.orchestra.run.trial006.stdout:(14/37): python3-rgw-20.2.0-670.gadc19233.el9.x 2.9 MB/s | 99 kB 00:00 2026-02-18T06:33:51.708 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:33:51.719 INFO:teuthology.orchestra.run.trial051.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/39 2026-02-18T06:33:51.872 INFO:teuthology.orchestra.run.trial051.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/39 2026-02-18T06:33:51.878 INFO:teuthology.orchestra.run.trial051.stdout: Upgrading : librados2-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:33:51.910 INFO:teuthology.orchestra.run.trial006.stdout:(15/37): mailcap-2.1.49-5.el9.noarch.rpm 120 kB/s | 33 kB 00:00 2026-02-18T06:33:51.929 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librados2-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:33:51.930 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:33:51.943 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:33:51.945 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 5/39 2026-02-18T06:33:51.946 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:33:51.973 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:33:51.975 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 7/39 2026-02-18T06:33:51.982 INFO:teuthology.orchestra.run.trial051.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/39 2026-02-18T06:33:51.984 INFO:teuthology.orchestra.run.trial006.stdout:(16/37): ceph-radosgw-20.2.0-670.gadc19233.el9. 12 MB/s | 24 MB 00:02 2026-02-18T06:33:51.985 INFO:teuthology.orchestra.run.trial051.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 9/39 2026-02-18T06:33:51.988 INFO:teuthology.orchestra.run.trial051.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 10/39 2026-02-18T06:33:51.992 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 11/39 2026-02-18T06:33:52.001 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x 12/39 2026-02-18T06:33:52.014 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 13/39 2026-02-18T06:33:52.015 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:33:52.028 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:33:52.058 INFO:teuthology.orchestra.run.trial051.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/39 2026-02-18T06:33:52.062 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/39 2026-02-18T06:33:52.068 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/39 2026-02-18T06:33:52.070 INFO:teuthology.orchestra.run.trial051.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/39 2026-02-18T06:33:52.075 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 19/39 2026-02-18T06:33:52.140 INFO:teuthology.orchestra.run.trial006.stdout:(17/37): pciutils-3.7.0-7.el9.x86_64.rpm 405 kB/s | 93 kB 00:00 2026-02-18T06:33:52.198 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 20/39 2026-02-18T06:33:52.199 INFO:teuthology.orchestra.run.trial051.stdout: Upgrading : librbd1-2:20.2.0-670.gadc19233.el9.x86_64 21/39 2026-02-18T06:33:52.225 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librbd1-2:20.2.0-670.gadc19233.el9.x86_64 21/39 2026-02-18T06:33:52.227 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 22/39 2026-02-18T06:33:52.235 INFO:teuthology.orchestra.run.trial051.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 23/39 2026-02-18T06:33:52.343 INFO:teuthology.orchestra.run.trial006.stdout:(18/37): nvme-cli-2.13-1.el9.x86_64.rpm 1.6 MB/s | 1.1 MB 00:00 2026-02-18T06:33:52.515 INFO:teuthology.orchestra.run.trial051.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 24/39 2026-02-18T06:33:52.516 INFO:teuthology.orchestra.run.trial051.stdout: Installing : librgw2-2:20.2.0-670.gadc19233.el9.x86_64 25/39 2026-02-18T06:33:52.532 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librgw2-2:20.2.0-670.gadc19233.el9.x86_64 25/39 2026-02-18T06:33:52.535 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 26/39 2026-02-18T06:33:52.540 INFO:teuthology.orchestra.run.trial051.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 27/39 2026-02-18T06:33:52.542 INFO:teuthology.orchestra.run.trial051.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 28/39 2026-02-18T06:33:52.543 INFO:teuthology.orchestra.run.trial051.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 29/39 2026-02-18T06:33:52.551 INFO:teuthology.orchestra.run.trial006.stdout:(19/37): boost-program-options-1.75.0-13.el9.x8 185 kB/s | 104 kB 00:00 2026-02-18T06:33:52.560 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 29/39 2026-02-18T06:33:52.630 INFO:teuthology.orchestra.run.trial006.stdout:(20/37): librabbitmq-0.11.0-7.el9.x86_64.rpm 546 kB/s | 45 kB 00:00 2026-02-18T06:33:52.670 INFO:teuthology.orchestra.run.trial051.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 30/39 2026-02-18T06:33:52.673 INFO:teuthology.orchestra.run.trial051.stdout: Installing : nvme-cli-2.13-1.el9.x86_64 31/39 2026-02-18T06:33:52.780 INFO:teuthology.orchestra.run.trial006.stdout:(21/37): libnbd-1.20.3-4.el9.x86_64.rpm 256 kB/s | 164 kB 00:00 2026-02-18T06:33:52.903 INFO:teuthology.orchestra.run.trial006.stdout:(22/37): libpmemobj-1.12.1-1.el9.x86_64.rpm 286 kB/s | 160 kB 00:00 2026-02-18T06:33:52.942 INFO:teuthology.orchestra.run.trial006.stdout:(23/37): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.8 MB/s | 292 kB 00:00 2026-02-18T06:33:52.958 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: nvme-cli-2.13-1.el9.x86_64 31/39 2026-02-18T06:33:52.958 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-18T06:33:52.958 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:53.002 INFO:teuthology.orchestra.run.trial051.stdout: Installing : mailcap-2.1.49-5.el9.noarch 32/39 2026-02-18T06:33:53.004 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86 33/39 2026-02-18T06:33:53.027 INFO:teuthology.orchestra.run.trial006.stdout:(24/37): qatlib-service-25.08.0-2.el9.x86_64.rp 441 kB/s | 37 kB 00:00 2026-02-18T06:33:53.112 INFO:teuthology.orchestra.run.trial006.stdout:(25/37): qatzip-libs-1.3.1-1.el9.x86_64.rpm 785 kB/s | 66 kB 00:00 2026-02-18T06:33:53.144 INFO:teuthology.orchestra.run.trial006.stdout:(26/37): gperftools-libs-2.9.1-3.el9.x86_64.rpm 9.2 MB/s | 308 kB 00:00 2026-02-18T06:33:53.164 INFO:teuthology.orchestra.run.trial006.stdout:(27/37): libarrow-9.0.0-15.el9.x86_64.rpm 230 MB/s | 4.4 MB 00:00 2026-02-18T06:33:53.167 INFO:teuthology.orchestra.run.trial006.stdout:(28/37): libarrow-doc-9.0.0-15.el9.noarch.rpm 7.2 MB/s | 25 kB 00:00 2026-02-18T06:33:53.171 INFO:teuthology.orchestra.run.trial006.stdout:(29/37): liboath-2.6.12-1.el9.x86_64.rpm 13 MB/s | 49 kB 00:00 2026-02-18T06:33:53.175 INFO:teuthology.orchestra.run.trial006.stdout:(30/37): libunwind-1.6.2-1.el9.x86_64.rpm 19 MB/s | 67 kB 00:00 2026-02-18T06:33:53.180 INFO:teuthology.orchestra.run.trial006.stdout:(31/37): parquet-libs-9.0.0-15.el9.x86_64.rpm 151 MB/s | 838 kB 00:00 2026-02-18T06:33:53.184 INFO:teuthology.orchestra.run.trial006.stdout:(32/37): re2-20211101-20.el9.x86_64.rpm 49 MB/s | 191 kB 00:00 2026-02-18T06:33:53.191 INFO:teuthology.orchestra.run.trial006.stdout:(33/37): thrift-0.15.0-4.el9.x86_64.rpm 223 MB/s | 1.6 MB 00:00 2026-02-18T06:33:53.221 INFO:teuthology.orchestra.run.trial006.stdout:(34/37): qatlib-25.08.0-2.el9.x86_64.rpm 753 kB/s | 240 kB 00:00 2026-02-18T06:33:53.331 INFO:teuthology.orchestra.run.trial006.stdout:(35/37): librados2-20.2.0-670.gadc19233.el9.x86 25 MB/s | 3.5 MB 00:00 2026-02-18T06:33:53.350 INFO:teuthology.orchestra.run.trial006.stdout:(36/37): librbd1-20.2.0-670.gadc19233.el9.x86_6 22 MB/s | 2.8 MB 00:00 2026-02-18T06:33:53.419 INFO:teuthology.orchestra.run.trial006.stdout:(37/37): librdkafka-1.6.1-102.el9.x86_64.rpm 840 kB/s | 662 kB 00:00 2026-02-18T06:33:53.425 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:33:53.425 INFO:teuthology.orchestra.run.trial006.stdout:Total 20 MB/s | 79 MB 00:04 2026-02-18T06:33:53.530 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:33:53.549 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:33:53.549 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:33:53.845 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:33:53.846 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:33:54.079 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:54.082 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:54.188 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:33:54.199 INFO:teuthology.orchestra.run.trial006.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/39 2026-02-18T06:33:54.355 INFO:teuthology.orchestra.run.trial006.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/39 2026-02-18T06:33:54.359 INFO:teuthology.orchestra.run.trial006.stdout: Upgrading : librados2-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:33:54.376 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:54.380 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 35/39 2026-02-18T06:33:54.411 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librados2-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:33:54.412 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:33:54.419 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 35/39 2026-02-18T06:33:54.419 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-18T06:33:54.420 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-18T06:33:54.420 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:54.423 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:33:54.426 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:33:54.428 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 5/39 2026-02-18T06:33:54.429 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:33:54.458 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:33:54.460 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 7/39 2026-02-18T06:33:54.469 INFO:teuthology.orchestra.run.trial006.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/39 2026-02-18T06:33:54.471 INFO:teuthology.orchestra.run.trial006.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 9/39 2026-02-18T06:33:54.474 INFO:teuthology.orchestra.run.trial006.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 10/39 2026-02-18T06:33:54.479 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 11/39 2026-02-18T06:33:54.488 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x 12/39 2026-02-18T06:33:54.499 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 13/39 2026-02-18T06:33:54.500 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:33:54.514 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:33:54.546 INFO:teuthology.orchestra.run.trial006.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/39 2026-02-18T06:33:54.551 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 16/39 2026-02-18T06:33:54.558 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/39 2026-02-18T06:33:54.560 INFO:teuthology.orchestra.run.trial006.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/39 2026-02-18T06:33:54.564 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 19/39 2026-02-18T06:33:54.694 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 20/39 2026-02-18T06:33:54.695 INFO:teuthology.orchestra.run.trial006.stdout: Upgrading : librbd1-2:20.2.0-670.gadc19233.el9.x86_64 21/39 2026-02-18T06:33:54.722 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librbd1-2:20.2.0-670.gadc19233.el9.x86_64 21/39 2026-02-18T06:33:54.724 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 22/39 2026-02-18T06:33:54.730 INFO:teuthology.orchestra.run.trial006.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 23/39 2026-02-18T06:33:55.013 INFO:teuthology.orchestra.run.trial006.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 24/39 2026-02-18T06:33:55.014 INFO:teuthology.orchestra.run.trial006.stdout: Installing : librgw2-2:20.2.0-670.gadc19233.el9.x86_64 25/39 2026-02-18T06:33:55.031 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librgw2-2:20.2.0-670.gadc19233.el9.x86_64 25/39 2026-02-18T06:33:55.034 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 26/39 2026-02-18T06:33:55.039 INFO:teuthology.orchestra.run.trial006.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 27/39 2026-02-18T06:33:55.041 INFO:teuthology.orchestra.run.trial006.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 28/39 2026-02-18T06:33:55.042 INFO:teuthology.orchestra.run.trial006.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 29/39 2026-02-18T06:33:55.060 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 29/39 2026-02-18T06:33:55.173 INFO:teuthology.orchestra.run.trial006.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 30/39 2026-02-18T06:33:55.176 INFO:teuthology.orchestra.run.trial006.stdout: Installing : nvme-cli-2.13-1.el9.x86_64 31/39 2026-02-18T06:33:55.467 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: nvme-cli-2.13-1.el9.x86_64 31/39 2026-02-18T06:33:55.467 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-18T06:33:55.467 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:33:55.508 INFO:teuthology.orchestra.run.trial006.stdout: Installing : mailcap-2.1.49-5.el9.noarch 32/39 2026-02-18T06:33:55.510 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86 33/39 2026-02-18T06:33:56.610 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:56.613 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:56.917 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 34/39 2026-02-18T06:33:56.921 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 35/39 2026-02-18T06:33:56.966 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 35/39 2026-02-18T06:33:56.966 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-18T06:33:56.966 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-18T06:33:56.966 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:33:56.970 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:33:58.943 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:33:58.943 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /sys 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /proc 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /mnt 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /var/tmp 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /home 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /root 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout:skipping the directory /tmp 2026-02-18T06:33:58.944 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:59.968 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 37/39 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 37/39 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-18T06:33:59.991 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:33:59.993 INFO:teuthology.orchestra.run.trial051.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 38/39 2026-02-18T06:34:00.005 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 38/39 2026-02-18T06:34:00.005 INFO:teuthology.orchestra.run.trial051.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:00.061 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:00.061 INFO:teuthology.orchestra.run.trial051.stdout:Creating group 'qat' with GID 992. 2026-02-18T06:34:00.061 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 1/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 5/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 7/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 8/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librgw2-2:20.2.0-670.gadc19233.el9.x86_64 9/39 2026-02-18T06:34:00.366 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x 10/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86 11/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 12/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 13/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 15/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 16/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : nvme-cli-2.13-1.el9.x86_64 17/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 18/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 19/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 20/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 21/39 2026-02-18T06:34:00.367 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 22/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 23/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 24/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 25/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 26/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 27/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 28/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 29/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 30/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 31/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 32/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 33/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : re2-1:20211101-20.el9.x86_64 34/39 2026-02-18T06:34:00.368 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 35/39 2026-02-18T06:34:00.369 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librados2-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:34:00.369 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 37/39 2026-02-18T06:34:00.369 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librbd1-2:20.2.0-670.gadc19233.el9.x86_64 38/39 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout:Upgraded: 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: librados2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: librbd1-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.421 INFO:teuthology.orchestra.run.trial051.stdout: ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-18T06:34:00.422 INFO:teuthology.orchestra.run.trial051.stdout: libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: librgw2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: nvme-cli-2.13-1.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.423 INFO:teuthology.orchestra.run.trial051.stdout: python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: re2-1:20211101-20.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:00.424 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:00.466 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-test 2026-02-18T06:34:00.780 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:13 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:00.861 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:00.862 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repository Size 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: ceph-test x86_64 2:20.2.0-670.gadc19233.el9 ceph 84 M 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout:Installing dependencies: 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: libcephsqlite x86_64 2:20.2.0-670.gadc19233.el9 ceph 164 k 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-18T06:34:00.863 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:Install 5 Packages 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 85 M 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 364 M 2026-02-18T06:34:00.864 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:01.258 INFO:teuthology.orchestra.run.trial051.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 1.7 MB/s | 233 kB 00:00 2026-02-18T06:34:01.329 INFO:teuthology.orchestra.run.trial051.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 4.2 MB/s | 303 kB 00:00 2026-02-18T06:34:01.378 INFO:teuthology.orchestra.run.trial051.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.3 MB/s | 64 kB 00:00 2026-02-18T06:34:01.492 INFO:teuthology.orchestra.run.trial051.stdout:(4/5): libcephsqlite-20.2.0-670.gadc19233.el9.x 445 kB/s | 164 kB 00:00 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /sys 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /proc 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /mnt 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /var/tmp 2026-02-18T06:34:01.505 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /home 2026-02-18T06:34:01.506 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /root 2026-02-18T06:34:01.506 INFO:teuthology.orchestra.run.trial006.stdout:skipping the directory /tmp 2026-02-18T06:34:01.506 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:02.550 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 37/39 2026-02-18T06:34:02.573 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 37/39 2026-02-18T06:34:02.574 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:02.574 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-18T06:34:02.574 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-18T06:34:02.574 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-18T06:34:02.574 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:02.575 INFO:teuthology.orchestra.run.trial006.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 38/39 2026-02-18T06:34:02.588 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 38/39 2026-02-18T06:34:02.588 INFO:teuthology.orchestra.run.trial006.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:02.644 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:02.644 INFO:teuthology.orchestra.run.trial006.stdout:Creating group 'qat' with GID 992. 2026-02-18T06:34:02.644 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:02.970 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 1/39 2026-02-18T06:34:02.970 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 3/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 4/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 5/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 6/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 7/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 8/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librgw2-2:20.2.0-670.gadc19233.el9.x86_64 9/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x 10/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86 11/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 12/39 2026-02-18T06:34:02.971 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 13/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 14/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 15/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 16/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : nvme-cli-2.13-1.el9.x86_64 17/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 18/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 19/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 20/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 21/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 22/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 23/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 24/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 25/39 2026-02-18T06:34:02.972 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 26/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 27/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 28/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 29/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 30/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 31/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 32/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 33/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : re2-1:20211101-20.el9.x86_64 34/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 35/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librados2-2:20.2.0-670.gadc19233.el9.x86_64 36/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 37/39 2026-02-18T06:34:02.973 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librbd1-2:20.2.0-670.gadc19233.el9.x86_64 38/39 2026-02-18T06:34:03.027 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 39/39 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout:Upgraded: 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: librados2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: librbd1-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: ceph-radosgw-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.028 INFO:teuthology.orchestra.run.trial006.stdout: ceph-selinux-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-daemon-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-proxy2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: libradosstriper1-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.029 INFO:teuthology.orchestra.run.trial006.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: librgw2-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: nvme-cli-2.13-1.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: python3-ceph-argparse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: python3-ceph-common-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.030 INFO:teuthology.orchestra.run.trial006.stdout: python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: re2-1:20211101-20.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:03.031 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:03.073 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-test 2026-02-18T06:34:03.382 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:15 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:03.463 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:03.464 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:03.464 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repository Size 2026-02-18T06:34:03.464 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: ceph-test x86_64 2:20.2.0-670.gadc19233.el9 ceph 84 M 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout:Installing dependencies: 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: libcephsqlite x86_64 2:20.2.0-670.gadc19233.el9 ceph 164 k 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:03.465 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:03.466 INFO:teuthology.orchestra.run.trial006.stdout:Install 5 Packages 2026-02-18T06:34:03.466 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:03.466 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 85 M 2026-02-18T06:34:03.466 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 364 M 2026-02-18T06:34:03.466 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:04.016 INFO:teuthology.orchestra.run.trial006.stdout:(1/5): libcephsqlite-20.2.0-670.gadc19233.el9.x 342 kB/s | 164 kB 00:00 2026-02-18T06:34:04.481 INFO:teuthology.orchestra.run.trial006.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 247 kB/s | 233 kB 00:00 2026-02-18T06:34:04.716 INFO:teuthology.orchestra.run.trial006.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 271 kB/s | 64 kB 00:00 2026-02-18T06:34:04.815 INFO:teuthology.orchestra.run.trial006.stdout:(4/5): socat-1.7.4.1-8.el9.x86_64.rpm 380 kB/s | 303 kB 00:00 2026-02-18T06:34:10.205 INFO:teuthology.orchestra.run.trial051.stdout:(5/5): ceph-test-20.2.0-670.gadc19233.el9.x86_6 9.2 MB/s | 84 MB 00:09 2026-02-18T06:34:10.206 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:10.206 INFO:teuthology.orchestra.run.trial051.stdout:Total 9.1 MB/s | 85 MB 00:09 2026-02-18T06:34:10.232 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:10.241 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:10.242 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:10.529 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:10.529 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:10.839 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:10.844 INFO:teuthology.orchestra.run.trial051.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-18T06:34:10.848 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-18T06:34:10.856 INFO:teuthology.orchestra.run.trial051.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-18T06:34:10.857 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 4/5 2026-02-18T06:34:14.049 INFO:teuthology.orchestra.run.trial006.stdout:(5/5): ceph-test-20.2.0-670.gadc19233.el9.x86_6 8.0 MB/s | 84 MB 00:10 2026-02-18T06:34:14.051 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:14.051 INFO:teuthology.orchestra.run.trial006.stdout:Total 8.0 MB/s | 85 MB 00:10 2026-02-18T06:34:14.076 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:14.086 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:14.087 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:14.373 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:14.373 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:14.583 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 4/5 2026-02-18T06:34:14.601 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 5/5 2026-02-18T06:34:14.682 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:14.687 INFO:teuthology.orchestra.run.trial006.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-18T06:34:14.691 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-18T06:34:14.699 INFO:teuthology.orchestra.run.trial006.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-18T06:34:14.700 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 4/5 2026-02-18T06:34:14.922 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 5/5 2026-02-18T06:34:14.922 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 1/5 2026-02-18T06:34:14.922 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 2/5 2026-02-18T06:34:14.923 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-18T06:34:14.923 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:14.975 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:15.013 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph 2026-02-18T06:34:15.324 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:28 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:15.409 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:15.412 INFO:teuthology.orchestra.run.trial051.stdout:====================================================================================== 2026-02-18T06:34:15.412 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repository Size 2026-02-18T06:34:15.412 INFO:teuthology.orchestra.run.trial051.stdout:====================================================================================== 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph x86_64 2:20.2.0-670.gadc19233.el9 ceph 6.5 k 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout:Installing dependencies: 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-grafana-dashboards noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 43 k 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mds x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.3 M 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr x86_64 2:20.2.0-670.gadc19233.el9 ceph 961 k 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-modules-core noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 290 k 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mon x86_64 2:20.2.0-670.gadc19233.el9 ceph 5.1 M 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-osd x86_64 2:20.2.0-670.gadc19233.el9 ceph 17 M 2026-02-18T06:34:15.413 INFO:teuthology.orchestra.run.trial051.stdout: ceph-prometheus-alerts noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 17 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: cephadm noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 1.0 M 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: cryptsetup x86_64 2.8.1-2.el9 baseos 350 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-18T06:34:15.414 INFO:teuthology.orchestra.run.trial051.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-18T06:34:15.415 INFO:teuthology.orchestra.run.trial051.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-devel x86_64 3.9.25-2.el9 appstream 244 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-18T06:34:15.416 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-18T06:34:15.417 INFO:teuthology.orchestra.run.trial051.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-18T06:34:15.418 INFO:teuthology.orchestra.run.trial051.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-18T06:34:15.419 INFO:teuthology.orchestra.run.trial051.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout:Installing weak dependencies: 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-cephadm noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 173 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-dashboard noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 11 M 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 7.4 M 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-k8sevents noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 22 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-rook noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 50 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: ceph-volume noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 297 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-18T06:34:15.420 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:15.421 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:15.421 INFO:teuthology.orchestra.run.trial051.stdout:====================================================================================== 2026-02-18T06:34:15.421 INFO:teuthology.orchestra.run.trial051.stdout:Install 95 Packages 2026-02-18T06:34:15.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:15.421 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 101 M 2026-02-18T06:34:15.422 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 561 M 2026-02-18T06:34:15.422 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:15.956 INFO:teuthology.orchestra.run.trial051.stdout:(1/95): ceph-20.2.0-670.gadc19233.el9.x86_64.rp 36 kB/s | 6.5 kB 00:00 2026-02-18T06:34:16.230 INFO:teuthology.orchestra.run.trial051.stdout:(2/95): ceph-mgr-20.2.0-670.gadc19233.el9.x86_6 2.1 MB/s | 961 kB 00:00 2026-02-18T06:34:16.483 INFO:teuthology.orchestra.run.trial051.stdout:(3/95): ceph-mds-20.2.0-670.gadc19233.el9.x86_6 3.3 MB/s | 2.3 MB 00:00 2026-02-18T06:34:16.532 INFO:teuthology.orchestra.run.trial051.stdout:(4/95): ceph-grafana-dashboards-20.2.0-670.gadc 899 kB/s | 43 kB 00:00 2026-02-18T06:34:16.582 INFO:teuthology.orchestra.run.trial051.stdout:(5/95): ceph-mgr-cephadm-20.2.0-670.gadc19233.e 3.3 MB/s | 173 kB 00:00 2026-02-18T06:34:17.812 INFO:teuthology.orchestra.run.trial051.stdout:(6/95): ceph-mgr-dashboard-20.2.0-670.gadc19233 8.6 MB/s | 11 MB 00:01 2026-02-18T06:34:17.906 INFO:teuthology.orchestra.run.trial051.stdout:(7/95): ceph-mon-20.2.0-670.gadc19233.el9.x86_6 2.6 MB/s | 5.1 MB 00:01 2026-02-18T06:34:17.960 INFO:teuthology.orchestra.run.trial051.stdout:(8/95): ceph-mgr-k8sevents-20.2.0-670.gadc19233 414 kB/s | 22 kB 00:00 2026-02-18T06:34:18.071 INFO:teuthology.orchestra.run.trial051.stdout:(9/95): ceph-mgr-modules-core-20.2.0-670.gadc19 2.5 MB/s | 290 kB 00:00 2026-02-18T06:34:18.127 INFO:teuthology.orchestra.run.trial051.stdout:(10/95): ceph-mgr-rook-20.2.0-670.gadc19233.el9 901 kB/s | 50 kB 00:00 2026-02-18T06:34:18.152 INFO:teuthology.orchestra.run.trial051.stdout:(11/95): ceph-osd-20.2.0-670.gadc19233.el9.x86_ 8.8 MB/s | 17 MB 00:01 2026-02-18T06:34:18.174 INFO:teuthology.orchestra.run.trial051.stdout:(12/95): ceph-prometheus-alerts-20.2.0-670.gadc 372 kB/s | 17 kB 00:00 2026-02-18T06:34:18.199 INFO:teuthology.orchestra.run.trial051.stdout:(13/95): ceph-volume-20.2.0-670.gadc19233.el9.n 6.1 MB/s | 297 kB 00:00 2026-02-18T06:34:18.375 INFO:teuthology.orchestra.run.trial051.stdout:(14/95): cryptsetup-2.8.1-2.el9.x86_64.rpm 2.0 MB/s | 350 kB 00:00 2026-02-18T06:34:18.401 INFO:teuthology.orchestra.run.trial051.stdout:(15/95): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.5 MB/s | 40 kB 00:00 2026-02-18T06:34:18.428 INFO:teuthology.orchestra.run.trial051.stdout:(16/95): libconfig-1.7.2-9.el9.x86_64.rpm 2.6 MB/s | 72 kB 00:00 2026-02-18T06:34:18.466 INFO:teuthology.orchestra.run.trial051.stdout:(17/95): cephadm-20.2.0-670.gadc19233.el9.noarc 3.4 MB/s | 1.0 MB 00:00 2026-02-18T06:34:18.474 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 4/5 2026-02-18T06:34:18.492 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 5/5 2026-02-18T06:34:18.495 INFO:teuthology.orchestra.run.trial051.stdout:(18/95): ceph-mgr-diskprediction-local-20.2.0-6 11 MB/s | 7.4 MB 00:00 2026-02-18T06:34:18.496 INFO:teuthology.orchestra.run.trial051.stdout:(19/95): libgfortran-11.5.0-14.el9.x86_64.rpm 11 MB/s | 794 kB 00:00 2026-02-18T06:34:18.551 INFO:teuthology.orchestra.run.trial051.stdout:(20/95): python3-cryptography-36.0.1-5.el9.x86_ 23 MB/s | 1.2 MB 00:00 2026-02-18T06:34:18.578 INFO:teuthology.orchestra.run.trial051.stdout:(21/95): python3-ply-3.11-14.el9.noarch.rpm 3.9 MB/s | 106 kB 00:00 2026-02-18T06:34:18.605 INFO:teuthology.orchestra.run.trial051.stdout:(22/95): python3-pycparser-2.20-6.el9.noarch.rp 4.9 MB/s | 135 kB 00:00 2026-02-18T06:34:18.616 INFO:teuthology.orchestra.run.trial051.stdout:(23/95): libquadmath-11.5.0-14.el9.x86_64.rpm 1.2 MB/s | 184 kB 00:00 2026-02-18T06:34:18.633 INFO:teuthology.orchestra.run.trial051.stdout:(24/95): python3-pyparsing-2.4.7-9.el9.noarch.r 5.4 MB/s | 150 kB 00:00 2026-02-18T06:34:18.651 INFO:teuthology.orchestra.run.trial051.stdout:(25/95): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.6 MB/s | 253 kB 00:00 2026-02-18T06:34:18.652 INFO:teuthology.orchestra.run.trial051.stdout:(26/95): python3-requests-2.25.1-10.el9.noarch. 3.4 MB/s | 126 kB 00:00 2026-02-18T06:34:18.662 INFO:teuthology.orchestra.run.trial051.stdout:(27/95): python3-urllib3-1.26.5-7.el9.noarch.rp 7.3 MB/s | 218 kB 00:00 2026-02-18T06:34:18.687 INFO:teuthology.orchestra.run.trial051.stdout:(28/95): unzip-6.0-59.el9.x86_64.rpm 4.9 MB/s | 182 kB 00:00 2026-02-18T06:34:18.700 INFO:teuthology.orchestra.run.trial051.stdout:(29/95): zip-3.0-35.el9.x86_64.rpm 5.4 MB/s | 266 kB 00:00 2026-02-18T06:34:18.706 INFO:teuthology.orchestra.run.trial051.stdout:(30/95): flexiblas-3.0.4-9.el9.x86_64.rpm 680 kB/s | 30 kB 00:00 2026-02-18T06:34:18.725 INFO:teuthology.orchestra.run.trial051.stdout:(31/95): flexiblas-openblas-openmp-3.0.4-9.el9. 609 kB/s | 15 kB 00:00 2026-02-18T06:34:18.736 INFO:teuthology.orchestra.run.trial051.stdout:(32/95): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 8.0 MB/s | 246 kB 00:00 2026-02-18T06:34:18.744 INFO:teuthology.orchestra.run.trial051.stdout:(33/95): openblas-0.3.29-1.el9.x86_64.rpm 5.0 MB/s | 42 kB 00:00 2026-02-18T06:34:18.760 INFO:teuthology.orchestra.run.trial051.stdout:(34/95): lua-5.4.4-4.el9.x86_64.rpm 5.3 MB/s | 188 kB 00:00 2026-02-18T06:34:18.777 INFO:teuthology.orchestra.run.trial051.stdout:(35/95): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 34 MB/s | 3.0 MB 00:00 2026-02-18T06:34:18.796 INFO:teuthology.orchestra.run.trial051.stdout:(36/95): protobuf-3.14.0-17.el9.x86_64.rpm 28 MB/s | 1.0 MB 00:00 2026-02-18T06:34:18.812 INFO:teuthology.orchestra.run.trial051.stdout:(37/95): python3-babel-2.9.1-2.el9.noarch.rpm 168 MB/s | 6.0 MB 00:00 2026-02-18T06:34:18.813 INFO:teuthology.orchestra.run.trial051.stdout:(38/95): python3-devel-3.9.25-2.el9.x86_64.rpm 14 MB/s | 244 kB 00:00 2026-02-18T06:34:18.817 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 5/5 2026-02-18T06:34:18.817 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 1/5 2026-02-18T06:34:18.817 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 2/5 2026-02-18T06:34:18.817 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-18T06:34:18.817 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-18T06:34:18.823 INFO:teuthology.orchestra.run.trial051.stdout:(39/95): python3-jmespath-1.0.1-1.el9.noarch.rp 4.7 MB/s | 48 kB 00:00 2026-02-18T06:34:18.827 INFO:teuthology.orchestra.run.trial051.stdout:(40/95): python3-jinja2-2.11.3-8.el9.noarch.rpm 17 MB/s | 249 kB 00:00 2026-02-18T06:34:18.836 INFO:teuthology.orchestra.run.trial051.stdout:(41/95): python3-libstoragemgmt-1.10.1-1.el9.x8 14 MB/s | 177 kB 00:00 2026-02-18T06:34:18.844 INFO:teuthology.orchestra.run.trial051.stdout:(42/95): python3-lxml-4.6.5-3.el9.x86_64.rpm 73 MB/s | 1.2 MB 00:00 2026-02-18T06:34:18.846 INFO:teuthology.orchestra.run.trial051.stdout:(43/95): python3-markupsafe-1.1.1-12.el9.x86_64 3.5 MB/s | 35 kB 00:00 2026-02-18T06:34:18.858 INFO:teuthology.orchestra.run.trial051.stdout:(44/95): python3-numpy-f2py-1.23.5-2.el9.x86_64 34 MB/s | 442 kB 00:00 2026-02-18T06:34:18.864 INFO:teuthology.orchestra.run.trial051.stdout:(45/95): openblas-openmp-0.3.29-1.el9.x86_64.rp 44 MB/s | 5.3 MB 00:00 2026-02-18T06:34:18.868 INFO:teuthology.orchestra.run.trial051.stdout:(46/95): python3-packaging-20.9-5.el9.noarch.rp 7.9 MB/s | 77 kB 00:00 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: ceph-test-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: libcephsqlite-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-18T06:34:18.869 INFO:teuthology.orchestra.run.trial006.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-18T06:34:18.870 INFO:teuthology.orchestra.run.trial006.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-18T06:34:18.870 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:18.870 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:18.876 INFO:teuthology.orchestra.run.trial051.stdout:(47/95): python3-protobuf-3.14.0-17.el9.noarch. 22 MB/s | 267 kB 00:00 2026-02-18T06:34:18.884 INFO:teuthology.orchestra.run.trial051.stdout:(48/95): python3-numpy-1.23.5-2.el9.x86_64.rpm 153 MB/s | 6.1 MB 00:00 2026-02-18T06:34:18.885 INFO:teuthology.orchestra.run.trial051.stdout:(49/95): python3-pyasn1-0.4.8-6.el9.noarch.rpm 9.5 MB/s | 159 kB 00:00 2026-02-18T06:34:18.888 INFO:teuthology.orchestra.run.trial051.stdout:(50/95): python3-pyasn1-modules-0.4.8-6.el9.noa 23 MB/s | 279 kB 00:00 2026-02-18T06:34:18.895 INFO:teuthology.orchestra.run.trial051.stdout:(51/95): python3-requests-oauthlib-1.3.0-12.el9 5.0 MB/s | 54 kB 00:00 2026-02-18T06:34:18.898 INFO:teuthology.orchestra.run.trial051.stdout:(52/95): python3-toml-0.10.2-6.el9.noarch.rpm 4.2 MB/s | 42 kB 00:00 2026-02-18T06:34:18.905 INFO:teuthology.orchestra.run.trial051.stdout:(53/95): xmlsec1-1.2.29-13.el9.x86_64.rpm 18 MB/s | 189 kB 00:00 2026-02-18T06:34:18.907 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph 2026-02-18T06:34:18.908 INFO:teuthology.orchestra.run.trial051.stdout:(54/95): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 8.9 MB/s | 90 kB 00:00 2026-02-18T06:34:19.218 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:31 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:19.289 INFO:teuthology.orchestra.run.trial051.stdout:(55/95): python3-scipy-1.9.3-2.el9.x86_64.rpm 48 MB/s | 19 MB 00:00 2026-02-18T06:34:19.303 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:19.306 INFO:teuthology.orchestra.run.trial006.stdout:====================================================================================== 2026-02-18T06:34:19.306 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repository Size 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout:====================================================================================== 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph x86_64 2:20.2.0-670.gadc19233.el9 ceph 6.5 k 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout:Installing dependencies: 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-grafana-dashboards noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 43 k 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mds x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.3 M 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr x86_64 2:20.2.0-670.gadc19233.el9 ceph 961 k 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-modules-core noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 290 k 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mon x86_64 2:20.2.0-670.gadc19233.el9 ceph 5.1 M 2026-02-18T06:34:19.307 INFO:teuthology.orchestra.run.trial006.stdout: ceph-osd x86_64 2:20.2.0-670.gadc19233.el9 ceph 17 M 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: ceph-prometheus-alerts noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 17 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: cephadm noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 1.0 M 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: cryptsetup x86_64 2.8.1-2.el9 baseos 350 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-18T06:34:19.308 INFO:teuthology.orchestra.run.trial006.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-18T06:34:19.309 INFO:teuthology.orchestra.run.trial006.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-devel x86_64 3.9.25-2.el9 appstream 244 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-18T06:34:19.310 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-18T06:34:19.311 INFO:teuthology.orchestra.run.trial006.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-18T06:34:19.312 INFO:teuthology.orchestra.run.trial006.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-18T06:34:19.313 INFO:teuthology.orchestra.run.trial006.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout:Installing weak dependencies: 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-cephadm noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 173 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-dashboard noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 11 M 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 7.4 M 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-k8sevents noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 22 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-rook noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 50 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: ceph-volume noarch 2:20.2.0-670.gadc19233.el9 ceph-noarch 297 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-18T06:34:19.314 INFO:teuthology.orchestra.run.trial006.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-18T06:34:19.315 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:19.315 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:19.315 INFO:teuthology.orchestra.run.trial006.stdout:====================================================================================== 2026-02-18T06:34:19.315 INFO:teuthology.orchestra.run.trial006.stdout:Install 95 Packages 2026-02-18T06:34:19.315 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:19.316 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 101 M 2026-02-18T06:34:19.316 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 561 M 2026-02-18T06:34:19.316 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:19.555 INFO:teuthology.orchestra.run.trial051.stdout:(56/95): abseil-cpp-20211102.0-4.el9.x86_64.rpm 2.0 MB/s | 551 kB 00:00 2026-02-18T06:34:19.586 INFO:teuthology.orchestra.run.trial051.stdout:(57/95): grpc-data-1.46.7-10.el9.noarch.rpm 615 kB/s | 19 kB 00:00 2026-02-18T06:34:19.627 INFO:teuthology.orchestra.run.trial051.stdout:(58/95): luarocks-3.9.2-5.el9.noarch.rpm 3.7 MB/s | 151 kB 00:00 2026-02-18T06:34:19.703 INFO:teuthology.orchestra.run.trial051.stdout:(59/95): python3-asyncssh-2.13.2-5.el9.noarch.r 7.0 MB/s | 548 kB 00:00 2026-02-18T06:34:19.715 INFO:teuthology.orchestra.run.trial051.stdout:(60/95): lua-devel-5.4.4-4.el9.x86_64.rpm 28 kB/s | 22 kB 00:00 2026-02-18T06:34:19.738 INFO:teuthology.orchestra.run.trial051.stdout:(61/95): python3-autocommand-2.2.2-8.el9.noarch 840 kB/s | 29 kB 00:00 2026-02-18T06:34:19.749 INFO:teuthology.orchestra.run.trial051.stdout:(62/95): python3-backports-tarfile-1.2.0-1.el9. 1.7 MB/s | 60 kB 00:00 2026-02-18T06:34:19.771 INFO:teuthology.orchestra.run.trial051.stdout:(63/95): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.3 MB/s | 43 kB 00:00 2026-02-18T06:34:19.784 INFO:teuthology.orchestra.run.trial051.stdout:(64/95): protobuf-compiler-3.14.0-17.el9.x86_64 984 kB/s | 862 kB 00:00 2026-02-18T06:34:19.798 INFO:teuthology.orchestra.run.trial051.stdout:(65/95): python3-cachetools-4.2.4-1.el9.noarch. 660 kB/s | 32 kB 00:00 2026-02-18T06:34:19.806 INFO:teuthology.orchestra.run.trial051.stdout:(66/95): python3-certifi-2023.05.07-4.el9.noarc 408 kB/s | 14 kB 00:00 2026-02-18T06:34:19.833 INFO:teuthology.orchestra.run.trial051.stdout:(67/95): python3-cheroot-10.0.1-4.el9.noarch.rp 3.5 MB/s | 173 kB 00:00 2026-02-18T06:34:19.855 INFO:teuthology.orchestra.run.trial051.stdout:(68/95): python3-cherrypy-18.6.1-2.el9.noarch.r 6.2 MB/s | 358 kB 00:00 2026-02-18T06:34:19.898 INFO:teuthology.orchestra.run.trial051.stdout:(69/95): python3-google-auth-2.45.0-1.el9.noarc 2.7 MB/s | 254 kB 00:00 2026-02-18T06:34:19.999 INFO:teuthology.orchestra.run.trial006.stdout:(1/95): ceph-20.2.0-670.gadc19233.el9.x86_64.rp 58 kB/s | 6.5 kB 00:00 2026-02-18T06:34:20.017 INFO:teuthology.orchestra.run.trial051.stdout:(70/95): python3-grpcio-1.46.7-10.el9.x86_64.rp 11 MB/s | 2.0 MB 00:00 2026-02-18T06:34:20.023 INFO:teuthology.orchestra.run.trial051.stdout:(71/95): python3-grpcio-tools-1.46.7-10.el9.x86 855 kB/s | 144 kB 00:00 2026-02-18T06:34:20.032 INFO:teuthology.orchestra.run.trial051.stdout:(72/95): python3-influxdb-5.3.1-1.el9.noarch.rp 1.0 MB/s | 139 kB 00:00 2026-02-18T06:34:20.290 INFO:teuthology.orchestra.run.trial006.stdout:(2/95): ceph-mgr-20.2.0-670.gadc19233.el9.x86_6 2.3 MB/s | 961 kB 00:00 2026-02-18T06:34:20.326 INFO:teuthology.orchestra.run.trial051.stdout:(73/95): python3-isodate-0.6.1-3.el9.noarch.rpm 181 kB/s | 56 kB 00:00 2026-02-18T06:34:20.327 INFO:teuthology.orchestra.run.trial051.stdout:(74/95): python3-jaraco-8.2.1-3.el9.noarch.rpm 35 kB/s | 11 kB 00:00 2026-02-18T06:34:20.327 INFO:teuthology.orchestra.run.trial051.stdout:(75/95): python3-jaraco-classes-3.2.1-5.el9.noa 60 kB/s | 18 kB 00:00 2026-02-18T06:34:20.359 INFO:teuthology.orchestra.run.trial051.stdout:(76/95): python3-jaraco-collections-3.0.0-8.el9 720 kB/s | 23 kB 00:00 2026-02-18T06:34:20.360 INFO:teuthology.orchestra.run.trial051.stdout:(77/95): python3-jaraco-context-6.0.1-3.el9.noa 596 kB/s | 20 kB 00:00 2026-02-18T06:34:20.361 INFO:teuthology.orchestra.run.trial051.stdout:(78/95): python3-jaraco-functools-3.5.0-2.el9.n 572 kB/s | 19 kB 00:00 2026-02-18T06:34:20.391 INFO:teuthology.orchestra.run.trial051.stdout:(79/95): python3-jaraco-text-4.0.0-2.el9.noarch 820 kB/s | 26 kB 00:00 2026-02-18T06:34:20.396 INFO:teuthology.orchestra.run.trial006.stdout:(3/95): ceph-mds-20.2.0-670.gadc19233.el9.x86_6 4.6 MB/s | 2.3 MB 00:00 2026-02-18T06:34:20.431 INFO:teuthology.orchestra.run.trial006.stdout:(4/95): ceph-grafana-dashboards-20.2.0-670.gadc 1.2 MB/s | 43 kB 00:00 2026-02-18T06:34:20.457 INFO:teuthology.orchestra.run.trial051.stdout:(80/95): python3-kubernetes-26.1.0-3.el9.noarch 11 MB/s | 1.0 MB 00:00 2026-02-18T06:34:20.457 INFO:teuthology.orchestra.run.trial051.stdout:(81/95): python3-more-itertools-8.12.0-2.el9.no 822 kB/s | 79 kB 00:00 2026-02-18T06:34:20.459 INFO:teuthology.orchestra.run.trial051.stdout:(82/95): python3-msgpack-1.0.3-2.el9.x86_64.rpm 1.2 MB/s | 86 kB 00:00 2026-02-18T06:34:20.470 INFO:teuthology.orchestra.run.trial006.stdout:(5/95): ceph-mgr-cephadm-20.2.0-670.gadc19233.e 4.4 MB/s | 173 kB 00:00 2026-02-18T06:34:20.498 INFO:teuthology.orchestra.run.trial051.stdout:(83/95): python3-natsort-7.1.1-5.el9.noarch.rpm 1.4 MB/s | 58 kB 00:00 2026-02-18T06:34:20.499 INFO:teuthology.orchestra.run.trial051.stdout:(84/95): python3-portend-3.1.0-2.el9.noarch.rpm 394 kB/s | 16 kB 00:00 2026-02-18T06:34:20.505 INFO:teuthology.orchestra.run.trial051.stdout:(85/95): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.9 MB/s | 90 kB 00:00 2026-02-18T06:34:20.530 INFO:teuthology.orchestra.run.trial051.stdout:(86/95): python3-repoze-lru-0.7-16.el9.noarch.r 946 kB/s | 31 kB 00:00 2026-02-18T06:34:20.541 INFO:teuthology.orchestra.run.trial051.stdout:(87/95): python3-routes-2.5.1-5.el9.noarch.rpm 4.4 MB/s | 188 kB 00:00 2026-02-18T06:34:20.544 INFO:teuthology.orchestra.run.trial051.stdout:(88/95): python3-rsa-4.9-2.el9.noarch.rpm 1.5 MB/s | 59 kB 00:00 2026-02-18T06:34:20.569 INFO:teuthology.orchestra.run.trial051.stdout:(89/95): python3-saml-1.16.0-1.el9.noarch.rpm 3.2 MB/s | 125 kB 00:00 2026-02-18T06:34:20.573 INFO:teuthology.orchestra.run.trial051.stdout:(90/95): python3-tempora-5.0.0-2.el9.noarch.rpm 1.1 MB/s | 36 kB 00:00 2026-02-18T06:34:20.579 INFO:teuthology.orchestra.run.trial051.stdout:(91/95): python3-typing-extensions-4.15.0-1.el9 2.4 MB/s | 86 kB 00:00 2026-02-18T06:34:20.604 INFO:teuthology.orchestra.run.trial051.stdout:(92/95): python3-websocket-client-1.2.3-2.el9.n 2.5 MB/s | 90 kB 00:00 2026-02-18T06:34:20.607 INFO:teuthology.orchestra.run.trial051.stdout:(93/95): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 1.4 MB/s | 48 kB 00:00 2026-02-18T06:34:20.611 INFO:teuthology.orchestra.run.trial051.stdout:(94/95): python3-xmltodict-0.12.0-15.el9.noarch 695 kB/s | 22 kB 00:00 2026-02-18T06:34:20.637 INFO:teuthology.orchestra.run.trial051.stdout:(95/95): python3-zc-lockfile-2.0-10.el9.noarch. 615 kB/s | 20 kB 00:00 2026-02-18T06:34:20.642 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:20.642 INFO:teuthology.orchestra.run.trial051.stdout:Total 19 MB/s | 101 MB 00:05 2026-02-18T06:34:21.062 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:21.063 INFO:teuthology.orchestra.run.trial006.stdout:(6/95): ceph-mon-20.2.0-670.gadc19233.el9.x86_6 4.7 MB/s | 5.1 MB 00:01 2026-02-18T06:34:21.098 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:21.098 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:21.424 INFO:teuthology.orchestra.run.trial006.stdout:(7/95): ceph-mgr-dashboard-20.2.0-670.gadc19233 11 MB/s | 11 MB 00:00 2026-02-18T06:34:21.475 INFO:teuthology.orchestra.run.trial006.stdout:(8/95): ceph-mgr-k8sevents-20.2.0-670.gadc19233 437 kB/s | 22 kB 00:00 2026-02-18T06:34:21.529 INFO:teuthology.orchestra.run.trial006.stdout:(9/95): ceph-mgr-modules-core-20.2.0-670.gadc19 5.2 MB/s | 290 kB 00:00 2026-02-18T06:34:21.530 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:21.530 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:21.581 INFO:teuthology.orchestra.run.trial006.stdout:(10/95): ceph-mgr-rook-20.2.0-670.gadc19233.el9 977 kB/s | 50 kB 00:00 2026-02-18T06:34:21.633 INFO:teuthology.orchestra.run.trial006.stdout:(11/95): ceph-prometheus-alerts-20.2.0-670.gadc 336 kB/s | 17 kB 00:00 2026-02-18T06:34:21.691 INFO:teuthology.orchestra.run.trial006.stdout:(12/95): ceph-volume-20.2.0-670.gadc19233.el9.n 5.0 MB/s | 297 kB 00:00 2026-02-18T06:34:21.818 INFO:teuthology.orchestra.run.trial006.stdout:(13/95): cephadm-20.2.0-670.gadc19233.el9.noarc 7.9 MB/s | 1.0 MB 00:00 2026-02-18T06:34:21.900 INFO:teuthology.orchestra.run.trial006.stdout:(14/95): ceph-mgr-diskprediction-local-20.2.0-6 8.8 MB/s | 7.4 MB 00:00 2026-02-18T06:34:21.987 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:21.989 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/95 2026-02-18T06:34:21.993 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/95 2026-02-18T06:34:22.014 INFO:teuthology.orchestra.run.trial051.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 3/95 2026-02-18T06:34:22.048 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 4/95 2026-02-18T06:34:22.057 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 5/95 2026-02-18T06:34:22.062 INFO:teuthology.orchestra.run.trial051.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 6/95 2026-02-18T06:34:22.065 INFO:teuthology.orchestra.run.trial051.stdout: Installing : lua-5.4.4-4.el9.x86_64 7/95 2026-02-18T06:34:22.069 INFO:teuthology.orchestra.run.trial051.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 8/95 2026-02-18T06:34:22.093 INFO:teuthology.orchestra.run.trial051.stdout: Installing : unzip-6.0-59.el9.x86_64 9/95 2026-02-18T06:34:22.106 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 10/95 2026-02-18T06:34:22.107 INFO:teuthology.orchestra.run.trial006.stdout:(15/95): ceph-osd-20.2.0-670.gadc19233.el9.x86_ 9.3 MB/s | 17 MB 00:01 2026-02-18T06:34:22.109 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 11/95 2026-02-18T06:34:22.115 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 12/95 2026-02-18T06:34:22.117 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 13/95 2026-02-18T06:34:22.128 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 14/95 2026-02-18T06:34:22.134 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 15/95 2026-02-18T06:34:22.158 INFO:teuthology.orchestra.run.trial051.stdout: Installing : zip-3.0-35.el9.x86_64 16/95 2026-02-18T06:34:22.161 INFO:teuthology.orchestra.run.trial051.stdout: Installing : luarocks-3.9.2-5.el9.noarch 17/95 2026-02-18T06:34:22.168 INFO:teuthology.orchestra.run.trial051.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 18/95 2026-02-18T06:34:22.220 INFO:teuthology.orchestra.run.trial051.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 19/95 2026-02-18T06:34:22.233 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 20/95 2026-02-18T06:34:22.235 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-rsa-4.9-2.el9.noarch 21/95 2026-02-18T06:34:22.240 INFO:teuthology.orchestra.run.trial051.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 22/95 2026-02-18T06:34:22.245 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 23/95 2026-02-18T06:34:22.250 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 24/95 2026-02-18T06:34:22.253 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 25/95 2026-02-18T06:34:22.267 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 26/95 2026-02-18T06:34:22.272 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 27/95 2026-02-18T06:34:22.277 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 28/95 2026-02-18T06:34:22.288 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 29/95 2026-02-18T06:34:22.297 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 30/95 2026-02-18T06:34:22.304 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 31/95 2026-02-18T06:34:22.323 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 32/95 2026-02-18T06:34:22.335 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 33/95 2026-02-18T06:34:22.350 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 34/95 2026-02-18T06:34:22.355 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 35/95 2026-02-18T06:34:22.363 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 36/95 2026-02-18T06:34:22.403 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 37/95 2026-02-18T06:34:22.586 INFO:teuthology.orchestra.run.trial006.stdout:(16/95): cryptsetup-2.8.1-2.el9.x86_64.rpm 456 kB/s | 350 kB 00:00 2026-02-18T06:34:22.731 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 38/95 2026-02-18T06:34:22.740 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 39/95 2026-02-18T06:34:22.744 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 40/95 2026-02-18T06:34:22.750 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 41/95 2026-02-18T06:34:22.752 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 42/95 2026-02-18T06:34:22.781 INFO:teuthology.orchestra.run.trial051.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 43/95 2026-02-18T06:34:22.825 INFO:teuthology.orchestra.run.trial051.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 44/95 2026-02-18T06:34:22.832 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 45/95 2026-02-18T06:34:22.835 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 46/95 2026-02-18T06:34:22.841 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 47/95 2026-02-18T06:34:22.845 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 48/95 2026-02-18T06:34:22.852 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 49/95 2026-02-18T06:34:22.855 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 50/95 2026-02-18T06:34:22.883 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 51/95 2026-02-18T06:34:22.891 INFO:teuthology.orchestra.run.trial006.stdout:(17/95): ledmon-libs-1.1.0-3.el9.x86_64.rpm 41 kB/s | 40 kB 00:00 2026-02-18T06:34:22.893 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 52/95 2026-02-18T06:34:22.901 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 53/95 2026-02-18T06:34:22.908 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 54/95 2026-02-18T06:34:22.943 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 55/95 2026-02-18T06:34:23.169 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-devel-3.9.25-2.el9.x86_64 56/95 2026-02-18T06:34:23.192 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 57/95 2026-02-18T06:34:23.196 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 58/95 2026-02-18T06:34:23.255 INFO:teuthology.orchestra.run.trial051.stdout: Installing : openblas-0.3.29-1.el9.x86_64 59/95 2026-02-18T06:34:23.257 INFO:teuthology.orchestra.run.trial051.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 60/95 2026-02-18T06:34:23.278 INFO:teuthology.orchestra.run.trial051.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 61/95 2026-02-18T06:34:23.384 INFO:teuthology.orchestra.run.trial006.stdout:(18/95): libconfig-1.7.2-9.el9.x86_64.rpm 56 kB/s | 72 kB 00:01 2026-02-18T06:34:23.436 INFO:teuthology.orchestra.run.trial006.stdout:(19/95): libgfortran-11.5.0-14.el9.x86_64.rpm 935 kB/s | 794 kB 00:00 2026-02-18T06:34:23.606 INFO:teuthology.orchestra.run.trial051.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 62/95 2026-02-18T06:34:23.680 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 63/95 2026-02-18T06:34:23.757 INFO:teuthology.orchestra.run.trial006.stdout:(20/95): libquadmath-11.5.0-14.el9.x86_64.rpm 213 kB/s | 184 kB 00:00 2026-02-18T06:34:24.303 INFO:teuthology.orchestra.run.trial006.stdout:(21/95): python3-cffi-1.14.5-5.el9.x86_64.rpm 276 kB/s | 253 kB 00:00 2026-02-18T06:34:24.337 INFO:teuthology.orchestra.run.trial006.stdout:(22/95): python3-cryptography-36.0.1-5.el9.x86_ 1.4 MB/s | 1.2 MB 00:00 2026-02-18T06:34:24.358 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 64/95 2026-02-18T06:34:24.375 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 65/95 2026-02-18T06:34:24.386 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 66/95 2026-02-18T06:34:24.392 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-packaging-20.9-5.el9.noarch 67/95 2026-02-18T06:34:24.406 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-ply-3.11-14.el9.noarch 68/95 2026-02-18T06:34:24.421 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 69/95 2026-02-18T06:34:24.498 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 70/95 2026-02-18T06:34:24.506 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 71/95 2026-02-18T06:34:24.530 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 72/95 2026-02-18T06:34:24.560 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 73/95 2026-02-18T06:34:24.582 INFO:teuthology.orchestra.run.trial006.stdout:(23/95): python3-ply-3.11-14.el9.noarch.rpm 129 kB/s | 106 kB 00:00 2026-02-18T06:34:24.611 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 74/95 2026-02-18T06:34:24.617 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 75/95 2026-02-18T06:34:24.621 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 76/95 2026-02-18T06:34:24.622 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 77/95 2026-02-18T06:34:24.640 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:24.640 INFO:teuthology.orchestra.run.trial051.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-18T06:34:24.641 INFO:teuthology.orchestra.run.trial051.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-18T06:34:24.641 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:24.649 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:24.677 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:24.677 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-18T06:34:24.677 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:24.714 INFO:teuthology.orchestra.run.trial051.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 79/95 2026-02-18T06:34:24.778 INFO:teuthology.orchestra.run.trial051.stdout: Installing : cryptsetup-2.8.1-2.el9.x86_64 80/95 2026-02-18T06:34:24.780 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 81/95 2026-02-18T06:34:24.794 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 81/95 2026-02-18T06:34:24.794 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:24.794 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-18T06:34:24.794 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:25.083 INFO:teuthology.orchestra.run.trial006.stdout:(24/95): python3-pycparser-2.20-6.el9.noarch.rp 173 kB/s | 135 kB 00:00 2026-02-18T06:34:25.130 INFO:teuthology.orchestra.run.trial006.stdout:(25/95): python3-pyparsing-2.4.7-9.el9.noarch.r 190 kB/s | 150 kB 00:00 2026-02-18T06:34:25.482 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 82/95 2026-02-18T06:34:25.506 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 82/95 2026-02-18T06:34:25.506 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:25.507 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-18T06:34:25.507 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-18T06:34:25.507 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-18T06:34:25.507 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:25.564 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: cephadm-2:20.2.0-670.gadc19233.el9.noarch 83/95 2026-02-18T06:34:25.566 INFO:teuthology.orchestra.run.trial051.stdout: Installing : cephadm-2:20.2.0-670.gadc19233.el9.noarch 83/95 2026-02-18T06:34:25.573 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9. 84/95 2026-02-18T06:34:25.596 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9 85/95 2026-02-18T06:34:25.597 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 86/95 2026-02-18T06:34:25.797 INFO:teuthology.orchestra.run.trial006.stdout:(26/95): python3-urllib3-1.26.5-7.el9.noarch.rp 305 kB/s | 218 kB 00:00 2026-02-18T06:34:25.972 INFO:teuthology.orchestra.run.trial006.stdout:(27/95): python3-requests-2.25.1-10.el9.noarch. 91 kB/s | 126 kB 00:01 2026-02-18T06:34:26.001 INFO:teuthology.orchestra.run.trial006.stdout:(28/95): flexiblas-3.0.4-9.el9.x86_64.rpm 1.0 MB/s | 30 kB 00:00 2026-02-18T06:34:26.019 INFO:teuthology.orchestra.run.trial006.stdout:(29/95): unzip-6.0-59.el9.x86_64.rpm 205 kB/s | 182 kB 00:00 2026-02-18T06:34:26.042 INFO:teuthology.orchestra.run.trial006.stdout:(30/95): flexiblas-openblas-openmp-3.0.4-9.el9. 644 kB/s | 15 kB 00:00 2026-02-18T06:34:26.061 INFO:teuthology.orchestra.run.trial006.stdout:(31/95): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 50 MB/s | 3.0 MB 00:00 2026-02-18T06:34:26.072 INFO:teuthology.orchestra.run.trial006.stdout:(32/95): lua-5.4.4-4.el9.x86_64.rpm 17 MB/s | 188 kB 00:00 2026-02-18T06:34:26.076 INFO:teuthology.orchestra.run.trial006.stdout:(33/95): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 7.1 MB/s | 246 kB 00:00 2026-02-18T06:34:26.080 INFO:teuthology.orchestra.run.trial006.stdout:(34/95): openblas-0.3.29-1.el9.x86_64.rpm 4.7 MB/s | 42 kB 00:00 2026-02-18T06:34:26.092 INFO:teuthology.orchestra.run.trial006.stdout:(35/95): protobuf-3.14.0-17.el9.x86_64.rpm 84 MB/s | 1.0 MB 00:00 2026-02-18T06:34:26.125 INFO:teuthology.orchestra.run.trial006.stdout:(36/95): python3-babel-2.9.1-2.el9.noarch.rpm 186 MB/s | 6.0 MB 00:00 2026-02-18T06:34:26.136 INFO:teuthology.orchestra.run.trial006.stdout:(37/95): python3-devel-3.9.25-2.el9.x86_64.rpm 21 MB/s | 244 kB 00:00 2026-02-18T06:34:26.148 INFO:teuthology.orchestra.run.trial006.stdout:(38/95): python3-jinja2-2.11.3-8.el9.noarch.rpm 21 MB/s | 249 kB 00:00 2026-02-18T06:34:26.157 INFO:teuthology.orchestra.run.trial006.stdout:(39/95): python3-jmespath-1.0.1-1.el9.noarch.rp 4.9 MB/s | 48 kB 00:00 2026-02-18T06:34:26.167 INFO:teuthology.orchestra.run.trial006.stdout:(40/95): python3-libstoragemgmt-1.10.1-1.el9.x8 17 MB/s | 177 kB 00:00 2026-02-18T06:34:26.181 INFO:teuthology.orchestra.run.trial006.stdout:(41/95): python3-lxml-4.6.5-3.el9.x86_64.rpm 89 MB/s | 1.2 MB 00:00 2026-02-18T06:34:26.194 INFO:teuthology.orchestra.run.trial006.stdout:(42/95): openblas-openmp-0.3.29-1.el9.x86_64.rp 45 MB/s | 5.3 MB 00:00 2026-02-18T06:34:26.194 INFO:teuthology.orchestra.run.trial006.stdout:(43/95): python3-markupsafe-1.1.1-12.el9.x86_64 2.7 MB/s | 35 kB 00:00 2026-02-18T06:34:26.205 INFO:teuthology.orchestra.run.trial006.stdout:(44/95): python3-numpy-f2py-1.23.5-2.el9.x86_64 40 MB/s | 442 kB 00:00 2026-02-18T06:34:26.216 INFO:teuthology.orchestra.run.trial006.stdout:(45/95): python3-packaging-20.9-5.el9.noarch.rp 6.9 MB/s | 77 kB 00:00 2026-02-18T06:34:26.228 INFO:teuthology.orchestra.run.trial006.stdout:(46/95): python3-numpy-1.23.5-2.el9.x86_64.rpm 177 MB/s | 6.1 MB 00:00 2026-02-18T06:34:26.229 INFO:teuthology.orchestra.run.trial006.stdout:(47/95): python3-protobuf-3.14.0-17.el9.noarch. 20 MB/s | 267 kB 00:00 2026-02-18T06:34:26.240 INFO:teuthology.orchestra.run.trial006.stdout:(48/95): python3-pyasn1-0.4.8-6.el9.noarch.rpm 14 MB/s | 159 kB 00:00 2026-02-18T06:34:26.241 INFO:teuthology.orchestra.run.trial006.stdout:(49/95): python3-pyasn1-modules-0.4.8-6.el9.noa 24 MB/s | 279 kB 00:00 2026-02-18T06:34:26.250 INFO:teuthology.orchestra.run.trial006.stdout:(50/95): python3-requests-oauthlib-1.3.0-12.el9 5.4 MB/s | 54 kB 00:00 2026-02-18T06:34:26.259 INFO:teuthology.orchestra.run.trial006.stdout:(51/95): python3-toml-0.10.2-6.el9.noarch.rpm 4.5 MB/s | 42 kB 00:00 2026-02-18T06:34:26.269 INFO:teuthology.orchestra.run.trial006.stdout:(52/95): xmlsec1-1.2.29-13.el9.x86_64.rpm 18 MB/s | 189 kB 00:00 2026-02-18T06:34:26.279 INFO:teuthology.orchestra.run.trial006.stdout:(53/95): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 9.4 MB/s | 90 kB 00:00 2026-02-18T06:34:26.332 INFO:teuthology.orchestra.run.trial006.stdout:(54/95): python3-scipy-1.9.3-2.el9.x86_64.rpm 212 MB/s | 19 MB 00:00 2026-02-18T06:34:26.353 INFO:teuthology.orchestra.run.trial006.stdout:(55/95): lua-devel-5.4.4-4.el9.x86_64.rpm 298 kB/s | 22 kB 00:00 2026-02-18T06:34:26.458 INFO:teuthology.orchestra.run.trial006.stdout:(56/95): protobuf-compiler-3.14.0-17.el9.x86_64 6.7 MB/s | 862 kB 00:00 2026-02-18T06:34:26.535 INFO:teuthology.orchestra.run.trial006.stdout:(57/95): abseil-cpp-20211102.0-4.el9.x86_64.rpm 3.0 MB/s | 551 kB 00:00 2026-02-18T06:34:26.539 INFO:teuthology.orchestra.run.trial006.stdout:(58/95): grpc-data-1.46.7-10.el9.noarch.rpm 239 kB/s | 19 kB 00:00 2026-02-18T06:34:26.558 INFO:teuthology.orchestra.run.trial006.stdout:(59/95): luarocks-3.9.2-5.el9.noarch.rpm 6.5 MB/s | 151 kB 00:00 2026-02-18T06:34:26.573 INFO:teuthology.orchestra.run.trial006.stdout:(60/95): zip-3.0-35.el9.x86_64.rpm 342 kB/s | 266 kB 00:00 2026-02-18T06:34:26.577 INFO:teuthology.orchestra.run.trial006.stdout:(61/95): python3-autocommand-2.2.2-8.el9.noarch 1.5 MB/s | 29 kB 00:00 2026-02-18T06:34:26.597 INFO:teuthology.orchestra.run.trial006.stdout:(62/95): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 2.1 MB/s | 43 kB 00:00 2026-02-18T06:34:26.617 INFO:teuthology.orchestra.run.trial006.stdout:(63/95): python3-cachetools-4.2.4-1.el9.noarch. 1.6 MB/s | 32 kB 00:00 2026-02-18T06:34:26.637 INFO:teuthology.orchestra.run.trial006.stdout:(64/95): python3-certifi-2023.05.07-4.el9.noarc 721 kB/s | 14 kB 00:00 2026-02-18T06:34:26.642 INFO:teuthology.orchestra.run.trial006.stdout:(65/95): python3-asyncssh-2.13.2-5.el9.noarch.r 5.2 MB/s | 548 kB 00:00 2026-02-18T06:34:26.661 INFO:teuthology.orchestra.run.trial006.stdout:(66/95): python3-cheroot-10.0.1-4.el9.noarch.rp 7.2 MB/s | 173 kB 00:00 2026-02-18T06:34:26.672 INFO:teuthology.orchestra.run.trial006.stdout:(67/95): python3-cherrypy-18.6.1-2.el9.noarch.r 12 MB/s | 358 kB 00:00 2026-02-18T06:34:26.679 INFO:teuthology.orchestra.run.trial006.stdout:(68/95): python3-backports-tarfile-1.2.0-1.el9. 566 kB/s | 60 kB 00:00 2026-02-18T06:34:26.695 INFO:teuthology.orchestra.run.trial006.stdout:(69/95): python3-google-auth-2.45.0-1.el9.noarc 7.3 MB/s | 254 kB 00:00 2026-02-18T06:34:26.726 INFO:teuthology.orchestra.run.trial006.stdout:(70/95): python3-influxdb-5.3.1-1.el9.noarch.rp 4.4 MB/s | 139 kB 00:00 2026-02-18T06:34:26.735 INFO:teuthology.orchestra.run.trial006.stdout:(71/95): python3-grpcio-tools-1.46.7-10.el9.x86 2.5 MB/s | 144 kB 00:00 2026-02-18T06:34:26.750 INFO:teuthology.orchestra.run.trial006.stdout:(72/95): python3-isodate-0.6.1-3.el9.noarch.rpm 2.3 MB/s | 56 kB 00:00 2026-02-18T06:34:26.756 INFO:teuthology.orchestra.run.trial006.stdout:(73/95): python3-jaraco-8.2.1-3.el9.noarch.rpm 507 kB/s | 11 kB 00:00 2026-02-18T06:34:26.757 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 86/95 2026-02-18T06:34:26.762 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 87/95 2026-02-18T06:34:26.772 INFO:teuthology.orchestra.run.trial006.stdout:(74/95): python3-jaraco-classes-3.2.1-5.el9.noa 786 kB/s | 18 kB 00:00 2026-02-18T06:34:26.778 INFO:teuthology.orchestra.run.trial006.stdout:(75/95): python3-jaraco-collections-3.0.0-8.el9 1.0 MB/s | 23 kB 00:00 2026-02-18T06:34:26.795 INFO:teuthology.orchestra.run.trial006.stdout:(76/95): python3-jaraco-context-6.0.1-3.el9.noa 878 kB/s | 20 kB 00:00 2026-02-18T06:34:26.799 INFO:teuthology.orchestra.run.trial006.stdout:(77/95): python3-grpcio-1.46.7-10.el9.x86_64.rp 16 MB/s | 2.0 MB 00:00 2026-02-18T06:34:26.800 INFO:teuthology.orchestra.run.trial006.stdout:(78/95): python3-jaraco-functools-3.5.0-2.el9.n 908 kB/s | 19 kB 00:00 2026-02-18T06:34:26.815 INFO:teuthology.orchestra.run.trial006.stdout:(79/95): python3-jaraco-text-4.0.0-2.el9.noarch 1.3 MB/s | 26 kB 00:00 2026-02-18T06:34:26.831 INFO:teuthology.orchestra.run.trial006.stdout:(80/95): python3-more-itertools-8.12.0-2.el9.no 2.5 MB/s | 79 kB 00:00 2026-02-18T06:34:26.838 INFO:teuthology.orchestra.run.trial006.stdout:(81/95): python3-msgpack-1.0.3-2.el9.x86_64.rpm 3.6 MB/s | 86 kB 00:00 2026-02-18T06:34:26.842 INFO:teuthology.orchestra.run.trial006.stdout:(82/95): python3-kubernetes-26.1.0-3.el9.noarch 24 MB/s | 1.0 MB 00:00 2026-02-18T06:34:26.857 INFO:teuthology.orchestra.run.trial006.stdout:(83/95): python3-natsort-7.1.1-5.el9.noarch.rpm 2.2 MB/s | 58 kB 00:00 2026-02-18T06:34:26.861 INFO:teuthology.orchestra.run.trial006.stdout:(84/95): python3-portend-3.1.0-2.el9.noarch.rpm 706 kB/s | 16 kB 00:00 2026-02-18T06:34:26.866 INFO:teuthology.orchestra.run.trial006.stdout:(85/95): python3-pyOpenSSL-21.0.0-1.el9.noarch. 3.7 MB/s | 90 kB 00:00 2026-02-18T06:34:26.882 INFO:teuthology.orchestra.run.trial006.stdout:(86/95): python3-repoze-lru-0.7-16.el9.noarch.r 1.2 MB/s | 31 kB 00:00 2026-02-18T06:34:26.888 INFO:teuthology.orchestra.run.trial006.stdout:(87/95): python3-routes-2.5.1-5.el9.noarch.rpm 7.1 MB/s | 188 kB 00:00 2026-02-18T06:34:26.888 INFO:teuthology.orchestra.run.trial006.stdout:(88/95): python3-rsa-4.9-2.el9.noarch.rpm 2.6 MB/s | 59 kB 00:00 2026-02-18T06:34:26.908 INFO:teuthology.orchestra.run.trial006.stdout:(89/95): python3-tempora-5.0.0-2.el9.noarch.rpm 1.7 MB/s | 36 kB 00:00 2026-02-18T06:34:26.911 INFO:teuthology.orchestra.run.trial006.stdout:(90/95): python3-typing-extensions-4.15.0-1.el9 3.7 MB/s | 86 kB 00:00 2026-02-18T06:34:26.916 INFO:teuthology.orchestra.run.trial006.stdout:(91/95): python3-saml-1.16.0-1.el9.noarch.rpm 3.6 MB/s | 125 kB 00:00 2026-02-18T06:34:26.932 INFO:teuthology.orchestra.run.trial006.stdout:(92/95): python3-websocket-client-1.2.3-2.el9.n 3.6 MB/s | 90 kB 00:00 2026-02-18T06:34:26.934 INFO:teuthology.orchestra.run.trial006.stdout:(93/95): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 2.1 MB/s | 48 kB 00:00 2026-02-18T06:34:26.941 INFO:teuthology.orchestra.run.trial006.stdout:(94/95): python3-xmltodict-0.12.0-15.el9.noarch 902 kB/s | 22 kB 00:00 2026-02-18T06:34:26.956 INFO:teuthology.orchestra.run.trial006.stdout:(95/95): python3-zc-lockfile-2.0-10.el9.noarch. 844 kB/s | 20 kB 00:00 2026-02-18T06:34:26.958 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:26.958 INFO:teuthology.orchestra.run.trial006.stdout:Total 13 MB/s | 101 MB 00:07 2026-02-18T06:34:27.220 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 87/95 2026-02-18T06:34:27.221 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 88/95 2026-02-18T06:34:27.233 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 88/95 2026-02-18T06:34:27.233 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 89/95 2026-02-18T06:34:27.292 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 89/95 2026-02-18T06:34:27.335 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.n 90/95 2026-02-18T06:34:27.336 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 91/95 2026-02-18T06:34:27.358 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 91/95 2026-02-18T06:34:27.358 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:27.358 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-18T06:34:27.358 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-18T06:34:27.358 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-18T06:34:27.359 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:27.369 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 92/95 2026-02-18T06:34:27.389 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:27.426 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:27.426 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:27.598 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 92/95 2026-02-18T06:34:27.599 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 93/95 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 93/95 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-18T06:34:27.621 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:27.719 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 94/95 2026-02-18T06:34:27.739 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 94/95 2026-02-18T06:34:27.740 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:27.740 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-18T06:34:27.740 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-18T06:34:27.740 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-18T06:34:27.740 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:27.759 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-2:20.2.0-670.gadc19233.el9.x86_64 95/95 2026-02-18T06:34:27.857 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:27.858 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:28.310 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:28.312 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/95 2026-02-18T06:34:28.315 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/95 2026-02-18T06:34:28.336 INFO:teuthology.orchestra.run.trial006.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 3/95 2026-02-18T06:34:28.369 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 4/95 2026-02-18T06:34:28.379 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 5/95 2026-02-18T06:34:28.384 INFO:teuthology.orchestra.run.trial006.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 6/95 2026-02-18T06:34:28.386 INFO:teuthology.orchestra.run.trial006.stdout: Installing : lua-5.4.4-4.el9.x86_64 7/95 2026-02-18T06:34:28.391 INFO:teuthology.orchestra.run.trial006.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 8/95 2026-02-18T06:34:28.414 INFO:teuthology.orchestra.run.trial006.stdout: Installing : unzip-6.0-59.el9.x86_64 9/95 2026-02-18T06:34:28.427 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 10/95 2026-02-18T06:34:28.430 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 11/95 2026-02-18T06:34:28.436 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 12/95 2026-02-18T06:34:28.438 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 13/95 2026-02-18T06:34:28.449 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 14/95 2026-02-18T06:34:28.455 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 15/95 2026-02-18T06:34:28.470 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-2:20.2.0-670.gadc19233.el9.x86_64 95/95 2026-02-18T06:34:28.470 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-2:20.2.0-670.gadc19233.el9.x86_64 1/95 2026-02-18T06:34:28.470 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 2/95 2026-02-18T06:34:28.470 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 3/95 2026-02-18T06:34:28.470 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 4/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 5/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9 6/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 7/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 8/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 9/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 10/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.n 11/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 12/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9. 13/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 14/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : cephadm-2:20.2.0-670.gadc19233.el9.noarch 15/95 2026-02-18T06:34:28.471 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : cryptsetup-2.8.1-2.el9.x86_64 16/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/95 2026-02-18T06:34:28.472 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : zip-3.0-35.el9.x86_64 29/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/95 2026-02-18T06:34:28.473 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-devel-3.9.25-2.el9.x86_64 39/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 41/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 42/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 43/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/95 2026-02-18T06:34:28.474 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/95 2026-02-18T06:34:28.475 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/95 2026-02-18T06:34:28.476 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 81/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 82/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 83/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 84/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 85/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 86/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 87/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 88/95 2026-02-18T06:34:28.477 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 89/95 2026-02-18T06:34:28.478 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 90/95 2026-02-18T06:34:28.478 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 91/95 2026-02-18T06:34:28.478 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 92/95 2026-02-18T06:34:28.478 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 93/95 2026-02-18T06:34:28.478 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 94/95 2026-02-18T06:34:28.479 INFO:teuthology.orchestra.run.trial006.stdout: Installing : zip-3.0-35.el9.x86_64 16/95 2026-02-18T06:34:28.483 INFO:teuthology.orchestra.run.trial006.stdout: Installing : luarocks-3.9.2-5.el9.noarch 17/95 2026-02-18T06:34:28.490 INFO:teuthology.orchestra.run.trial006.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 18/95 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 95/95 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: ceph-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:28.532 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-diskprediction-local-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: cephadm-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:28.533 INFO:teuthology.orchestra.run.trial051.stdout: cryptsetup-2.8.1-2.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: lua-5.4.4-4.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-18T06:34:28.534 INFO:teuthology.orchestra.run.trial051.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-18T06:34:28.535 INFO:teuthology.orchestra.run.trial051.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-devel-3.9.25-2.el9.x86_64 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-18T06:34:28.536 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-18T06:34:28.537 INFO:teuthology.orchestra.run.trial051.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-ply-3.11-14.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-18T06:34:28.538 INFO:teuthology.orchestra.run.trial051.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-18T06:34:28.539 INFO:teuthology.orchestra.run.trial051.stdout: unzip-6.0-59.el9.x86_64 2026-02-18T06:34:28.540 INFO:teuthology.orchestra.run.trial051.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-18T06:34:28.540 INFO:teuthology.orchestra.run.trial051.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-18T06:34:28.540 INFO:teuthology.orchestra.run.trial051.stdout: zip-3.0-35.el9.x86_64 2026-02-18T06:34:28.540 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:28.540 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:28.541 INFO:teuthology.orchestra.run.trial006.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 19/95 2026-02-18T06:34:28.554 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 20/95 2026-02-18T06:34:28.556 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-rsa-4.9-2.el9.noarch 21/95 2026-02-18T06:34:28.560 INFO:teuthology.orchestra.run.trial006.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 22/95 2026-02-18T06:34:28.566 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 23/95 2026-02-18T06:34:28.571 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 24/95 2026-02-18T06:34:28.574 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 25/95 2026-02-18T06:34:28.588 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 26/95 2026-02-18T06:34:28.589 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-base 2026-02-18T06:34:28.593 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 27/95 2026-02-18T06:34:28.598 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 28/95 2026-02-18T06:34:28.609 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 29/95 2026-02-18T06:34:28.618 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 30/95 2026-02-18T06:34:28.625 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 31/95 2026-02-18T06:34:28.644 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 32/95 2026-02-18T06:34:28.656 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 33/95 2026-02-18T06:34:28.671 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 34/95 2026-02-18T06:34:28.676 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 35/95 2026-02-18T06:34:28.684 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 36/95 2026-02-18T06:34:28.724 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 37/95 2026-02-18T06:34:28.902 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:41 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:28.965 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:28.983 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:28.984 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:28.984 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:29.005 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install cephadm 2026-02-18T06:34:29.051 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 38/95 2026-02-18T06:34:29.061 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 39/95 2026-02-18T06:34:29.065 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 40/95 2026-02-18T06:34:29.071 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 41/95 2026-02-18T06:34:29.073 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 42/95 2026-02-18T06:34:29.101 INFO:teuthology.orchestra.run.trial006.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 43/95 2026-02-18T06:34:29.146 INFO:teuthology.orchestra.run.trial006.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 44/95 2026-02-18T06:34:29.152 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 45/95 2026-02-18T06:34:29.156 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 46/95 2026-02-18T06:34:29.162 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 47/95 2026-02-18T06:34:29.165 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 48/95 2026-02-18T06:34:29.172 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 49/95 2026-02-18T06:34:29.176 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 50/95 2026-02-18T06:34:29.204 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 51/95 2026-02-18T06:34:29.214 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 52/95 2026-02-18T06:34:29.221 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 53/95 2026-02-18T06:34:29.228 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 54/95 2026-02-18T06:34:29.263 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 55/95 2026-02-18T06:34:29.316 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:42 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:29.380 INFO:teuthology.orchestra.run.trial051.stdout:Package cephadm-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:29.397 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:29.398 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:29.398 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:29.420 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-immutable-object-cache 2026-02-18T06:34:29.489 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-devel-3.9.25-2.el9.x86_64 56/95 2026-02-18T06:34:29.512 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 57/95 2026-02-18T06:34:29.516 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 58/95 2026-02-18T06:34:29.575 INFO:teuthology.orchestra.run.trial006.stdout: Installing : openblas-0.3.29-1.el9.x86_64 59/95 2026-02-18T06:34:29.577 INFO:teuthology.orchestra.run.trial006.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 60/95 2026-02-18T06:34:29.598 INFO:teuthology.orchestra.run.trial006.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 61/95 2026-02-18T06:34:29.733 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:42 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repo Size 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:29.815 INFO:teuthology.orchestra.run.trial051.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-670.gadc19233.el9 ceph 154 k 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 154 k 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 447 k 2026-02-18T06:34:29.816 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:29.925 INFO:teuthology.orchestra.run.trial006.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 62/95 2026-02-18T06:34:29.999 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 63/95 2026-02-18T06:34:30.011 INFO:teuthology.orchestra.run.trial051.stdout:ceph-immutable-object-cache-20.2.0-670.gadc1923 786 kB/s | 154 kB 00:00 2026-02-18T06:34:30.012 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:30.012 INFO:teuthology.orchestra.run.trial051.stdout:Total 782 kB/s | 154 kB 00:00 2026-02-18T06:34:30.012 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:30.017 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:30.017 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:30.050 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:30.050 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:30.105 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:30.106 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:30.128 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:30.128 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:30.128 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-18T06:34:30.129 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:30.444 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:30.445 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:30.445 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:30.445 INFO:teuthology.orchestra.run.trial051.stdout: ceph-immutable-object-cache-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:30.445 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:30.445 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:30.472 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-mgr 2026-02-18T06:34:30.676 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 64/95 2026-02-18T06:34:30.693 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 65/95 2026-02-18T06:34:30.703 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 66/95 2026-02-18T06:34:30.710 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-packaging-20.9-5.el9.noarch 67/95 2026-02-18T06:34:30.724 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-ply-3.11-14.el9.noarch 68/95 2026-02-18T06:34:30.739 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 69/95 2026-02-18T06:34:30.780 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:43 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:30.815 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 70/95 2026-02-18T06:34:30.824 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 71/95 2026-02-18T06:34:30.844 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:30.848 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 72/95 2026-02-18T06:34:30.862 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:30.862 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:30.862 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:30.877 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 73/95 2026-02-18T06:34:30.883 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-mgr-dashboard 2026-02-18T06:34:30.929 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 74/95 2026-02-18T06:34:30.935 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 75/95 2026-02-18T06:34:30.938 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 76/95 2026-02-18T06:34:30.939 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 77/95 2026-02-18T06:34:30.957 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:30.958 INFO:teuthology.orchestra.run.trial006.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-18T06:34:30.958 INFO:teuthology.orchestra.run.trial006.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-18T06:34:30.958 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:30.965 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:30.991 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 78/95 2026-02-18T06:34:30.991 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-18T06:34:30.992 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:31.026 INFO:teuthology.orchestra.run.trial006.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 79/95 2026-02-18T06:34:31.090 INFO:teuthology.orchestra.run.trial006.stdout: Installing : cryptsetup-2.8.1-2.el9.x86_64 80/95 2026-02-18T06:34:31.092 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 81/95 2026-02-18T06:34:31.106 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 81/95 2026-02-18T06:34:31.107 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:31.107 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-18T06:34:31.107 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:31.191 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:44 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:31.256 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:31.274 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:31.275 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:31.275 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:31.296 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-18T06:34:31.607 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:44 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:31.671 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-mgr-diskprediction-local-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:31.689 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:31.690 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:31.690 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:31.712 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-mgr-rook 2026-02-18T06:34:31.780 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 82/95 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 82/95 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-18T06:34:31.806 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:31.867 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: cephadm-2:20.2.0-670.gadc19233.el9.noarch 83/95 2026-02-18T06:34:31.869 INFO:teuthology.orchestra.run.trial006.stdout: Installing : cephadm-2:20.2.0-670.gadc19233.el9.noarch 83/95 2026-02-18T06:34:31.876 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9. 84/95 2026-02-18T06:34:31.900 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9 85/95 2026-02-18T06:34:31.901 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 86/95 2026-02-18T06:34:32.020 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:45 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:32.085 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:32.102 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:32.103 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:32.103 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:32.124 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-mgr-cephadm 2026-02-18T06:34:32.439 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:45 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:32.502 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:32.520 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:32.520 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:32.520 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:32.543 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-fuse 2026-02-18T06:34:32.853 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:45 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:32.935 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:32.935 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout: ceph-fuse x86_64 2:20.2.0-670.gadc19233.el9 ceph 942 k 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 942 k 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 2.7 M 2026-02-18T06:34:32.936 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:33.053 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 86/95 2026-02-18T06:34:33.059 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 87/95 2026-02-18T06:34:33.226 INFO:teuthology.orchestra.run.trial051.stdout:ceph-fuse-20.2.0-670.gadc19233.el9.x86_64.rpm 3.2 MB/s | 942 kB 00:00 2026-02-18T06:34:33.226 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:33.226 INFO:teuthology.orchestra.run.trial051.stdout:Total 3.2 MB/s | 942 kB 00:00 2026-02-18T06:34:33.227 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:33.231 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:33.232 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:33.266 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:33.266 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:33.355 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:33.372 INFO:teuthology.orchestra.run.trial051.stdout: Installing : ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:33.516 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 87/95 2026-02-18T06:34:33.517 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 88/95 2026-02-18T06:34:33.531 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 88/95 2026-02-18T06:34:33.531 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 89/95 2026-02-18T06:34:33.577 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:33.590 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 89/95 2026-02-18T06:34:33.634 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.n 90/95 2026-02-18T06:34:33.634 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:33.635 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:33.635 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:33.635 INFO:teuthology.orchestra.run.trial051.stdout: ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:33.635 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:33.635 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:33.636 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 91/95 2026-02-18T06:34:33.656 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 91/95 2026-02-18T06:34:33.657 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:33.657 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-18T06:34:33.657 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-18T06:34:33.657 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-18T06:34:33.657 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:33.660 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install ceph-volume 2026-02-18T06:34:33.668 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 92/95 2026-02-18T06:34:33.889 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 92/95 2026-02-18T06:34:33.890 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 93/95 2026-02-18T06:34:33.912 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 93/95 2026-02-18T06:34:33.913 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:33.913 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-18T06:34:33.913 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-18T06:34:33.913 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-18T06:34:33.913 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:33.977 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:46 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:34.009 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 94/95 2026-02-18T06:34:34.029 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 94/95 2026-02-18T06:34:34.030 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:34.030 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-18T06:34:34.030 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-18T06:34:34.030 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-18T06:34:34.030 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:34.040 INFO:teuthology.orchestra.run.trial051.stdout:Package ceph-volume-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:34.048 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-2:20.2.0-670.gadc19233.el9.x86_64 95/95 2026-02-18T06:34:34.058 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:34.059 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:34.059 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:34.081 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install librados-devel 2026-02-18T06:34:34.393 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:47 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:34.473 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:34.474 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:34.474 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repo Size 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout: librados-devel x86_64 2:20.2.0-670.gadc19233.el9 ceph 126 k 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 126 k 2026-02-18T06:34:34.475 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 449 k 2026-02-18T06:34:34.476 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:34.674 INFO:teuthology.orchestra.run.trial051.stdout:librados-devel-20.2.0-670.gadc19233.el9.x86_64. 634 kB/s | 126 kB 00:00 2026-02-18T06:34:34.674 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:34.674 INFO:teuthology.orchestra.run.trial051.stdout:Total 629 kB/s | 126 kB 00:00 2026-02-18T06:34:34.675 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:34.678 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:34.678 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:34.709 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:34.709 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:34.747 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-2:20.2.0-670.gadc19233.el9.x86_64 95/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-2:20.2.0-670.gadc19233.el9.x86_64 1/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 2/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 3/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 4/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 5/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9 6/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 7/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noar 8/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-670.gadc192 9/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noar 10/95 2026-02-18T06:34:34.748 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.n 11/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 12/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9. 13/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 14/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : cephadm-2:20.2.0-670.gadc19233.el9.noarch 15/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : cryptsetup-2.8.1-2.el9.x86_64 16/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/95 2026-02-18T06:34:34.749 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : zip-3.0-35.el9.x86_64 29/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/95 2026-02-18T06:34:34.750 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-devel-3.9.25-2.el9.x86_64 39/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 41/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 42/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 43/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/95 2026-02-18T06:34:34.751 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/95 2026-02-18T06:34:34.752 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/95 2026-02-18T06:34:34.753 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 81/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 82/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 83/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 84/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 85/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 86/95 2026-02-18T06:34:34.754 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 87/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 88/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 89/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 90/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 91/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 92/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 93/95 2026-02-18T06:34:34.755 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 94/95 2026-02-18T06:34:34.763 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:34.780 INFO:teuthology.orchestra.run.trial051.stdout: Installing : librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 95/95 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: ceph-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: ceph-grafana-dashboards-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mds-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.810 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-diskprediction-local-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-k8sevents-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-modules-core-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-mon-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-osd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-prometheus-alerts-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: ceph-volume-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: cephadm-2:20.2.0-670.gadc19233.el9.noarch 2026-02-18T06:34:34.811 INFO:teuthology.orchestra.run.trial006.stdout: cryptsetup-2.8.1-2.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-18T06:34:34.812 INFO:teuthology.orchestra.run.trial006.stdout: lua-5.4.4-4.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-18T06:34:34.813 INFO:teuthology.orchestra.run.trial006.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-devel-3.9.25-2.el9.x86_64 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-18T06:34:34.814 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-18T06:34:34.815 INFO:teuthology.orchestra.run.trial006.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-ply-3.11-14.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-18T06:34:34.816 INFO:teuthology.orchestra.run.trial006.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-18T06:34:34.817 INFO:teuthology.orchestra.run.trial006.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout: unzip-6.0-59.el9.x86_64 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout: zip-3.0-35.el9.x86_64 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:34.818 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:34.858 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:34.864 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-base 2026-02-18T06:34:34.914 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:34.914 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:34.914 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:34.914 INFO:teuthology.orchestra.run.trial051.stdout: librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:34.915 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:34.915 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:34.943 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install libcephfs2 2026-02-18T06:34:35.178 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:47 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:35.241 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-base-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:35.256 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:48 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:35.259 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:35.260 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:35.261 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:35.281 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install cephadm 2026-02-18T06:34:35.322 INFO:teuthology.orchestra.run.trial051.stdout:Package libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:35.340 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:35.340 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:35.341 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:35.363 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install libcephfs-devel 2026-02-18T06:34:35.590 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:47 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:35.656 INFO:teuthology.orchestra.run.trial006.stdout:Package cephadm-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:35.673 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:35.674 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:35.674 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:35.675 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:48 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:35.694 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-immutable-object-cache 2026-02-18T06:34:35.755 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:35.756 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:35.756 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repo Size 2026-02-18T06:34:35.756 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-devel x86_64 2:20.2.0-670.gadc19233.el9 ceph 34 k 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 34 k 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 155 k 2026-02-18T06:34:35.757 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:35.901 INFO:teuthology.orchestra.run.trial051.stdout:libcephfs-devel-20.2.0-670.gadc19233.el9.x86_64 238 kB/s | 34 kB 00:00 2026-02-18T06:34:35.901 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:35.902 INFO:teuthology.orchestra.run.trial051.stdout:Total 236 kB/s | 34 kB 00:00 2026-02-18T06:34:35.902 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:35.904 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:35.905 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:35.907 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:35.907 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:35.928 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:35.943 INFO:teuthology.orchestra.run.trial051.stdout: Installing : libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:36.003 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:36.003 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:48 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:36.059 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:36.059 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:36.059 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:36.060 INFO:teuthology.orchestra.run.trial051.stdout: libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:36.060 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:36.060 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:36.084 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:36.085 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:36.085 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repo Size 2026-02-18T06:34:36.085 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-670.gadc19233.el9 ceph 154 k 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 154 k 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 447 k 2026-02-18T06:34:36.086 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:36.088 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install librados2 2026-02-18T06:34:36.282 INFO:teuthology.orchestra.run.trial006.stdout:ceph-immutable-object-cache-20.2.0-670.gadc1923 782 kB/s | 154 kB 00:00 2026-02-18T06:34:36.283 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:36.283 INFO:teuthology.orchestra.run.trial006.stdout:Total 778 kB/s | 154 kB 00:00 2026-02-18T06:34:36.283 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:36.288 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:36.289 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:36.321 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:36.321 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:36.376 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:36.378 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:36.401 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:36.402 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:36.402 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-18T06:34:36.402 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:36.402 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:49 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:36.465 INFO:teuthology.orchestra.run.trial051.stdout:Package librados2-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:36.483 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:36.483 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:36.484 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:36.505 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install librbd1 2026-02-18T06:34:36.712 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-670.gadc19233.e 1/1 2026-02-18T06:34:36.713 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:36.713 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:36.713 INFO:teuthology.orchestra.run.trial006.stdout: ceph-immutable-object-cache-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:36.713 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:36.713 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:36.741 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-mgr 2026-02-18T06:34:36.820 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:49 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:36.884 INFO:teuthology.orchestra.run.trial051.stdout:Package librbd1-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:36.902 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:36.903 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:36.903 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:36.925 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install python3-rados 2026-02-18T06:34:37.052 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:49 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:37.115 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-mgr-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:37.132 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:37.133 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:37.133 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:37.155 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-mgr-dashboard 2026-02-18T06:34:37.235 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:50 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:37.299 INFO:teuthology.orchestra.run.trial051.stdout:Package python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:37.317 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:37.317 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:37.317 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:37.339 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install python3-rgw 2026-02-18T06:34:37.463 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:49 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:37.526 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-mgr-dashboard-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:37.544 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:37.544 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:37.544 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:37.567 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-18T06:34:37.654 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:50 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:37.718 INFO:teuthology.orchestra.run.trial051.stdout:Package python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:37.736 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:37.737 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:37.737 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:37.759 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install python3-cephfs 2026-02-18T06:34:37.875 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:49 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:37.938 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-mgr-diskprediction-local-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:37.955 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:37.956 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:37.956 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:37.977 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-mgr-rook 2026-02-18T06:34:38.070 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:51 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:38.135 INFO:teuthology.orchestra.run.trial051.stdout:Package python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:38.153 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:38.153 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:38.153 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:38.174 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install python3-rbd 2026-02-18T06:34:38.287 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:50 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:38.351 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-mgr-rook-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:38.368 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:38.369 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:38.369 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:38.391 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-mgr-cephadm 2026-02-18T06:34:38.479 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:51 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:38.543 INFO:teuthology.orchestra.run.trial051.stdout:Package python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:38.561 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:38.561 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:38.561 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:38.582 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install rbd-fuse 2026-02-18T06:34:38.703 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:50 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:38.766 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-mgr-cephadm-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:38.784 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:38.785 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:38.785 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:38.807 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-fuse 2026-02-18T06:34:38.893 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:51 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:38.975 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:38.975 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout: rbd-fuse x86_64 2:20.2.0-670.gadc19233.el9 ceph 91 k 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:38.976 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:38.977 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 91 k 2026-02-18T06:34:38.977 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 238 k 2026-02-18T06:34:38.977 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:39.049 INFO:teuthology.orchestra.run.trial051.stdout:rbd-fuse-20.2.0-670.gadc19233.el9.x86_64.rpm 1.2 MB/s | 91 kB 00:00 2026-02-18T06:34:39.050 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:39.050 INFO:teuthology.orchestra.run.trial051.stdout:Total 1.2 MB/s | 91 kB 00:00 2026-02-18T06:34:39.050 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:39.054 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:39.054 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:39.087 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:39.087 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:39.119 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:51 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:39.139 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:39.155 INFO:teuthology.orchestra.run.trial051.stdout: Installing : rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.201 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:39.202 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:39.202 INFO:teuthology.orchestra.run.trial006.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:39.202 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:39.202 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:39.202 INFO:teuthology.orchestra.run.trial006.stdout: ceph-fuse x86_64 2:20.2.0-670.gadc19233.el9 ceph 942 k 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 942 k 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 2.7 M 2026-02-18T06:34:39.203 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:39.229 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout: rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:39.287 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:39.314 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install rbd-mirror 2026-02-18T06:34:39.320 INFO:teuthology.orchestra.run.trial006.stdout:ceph-fuse-20.2.0-670.gadc19233.el9.x86_64.rpm 7.8 MB/s | 942 kB 00:00 2026-02-18T06:34:39.321 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:39.321 INFO:teuthology.orchestra.run.trial006.stdout:Total 7.7 MB/s | 942 kB 00:00 2026-02-18T06:34:39.321 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:39.326 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:39.326 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:39.360 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:39.360 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:39.449 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:39.466 INFO:teuthology.orchestra.run.trial006.stdout: Installing : ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.625 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:52 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:39.708 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout: Package Arch Version Repo Size 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout: rbd-mirror x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.9 M 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 2.9 M 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 11 M 2026-02-18T06:34:39.709 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:39.713 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.769 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:39.770 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:39.770 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:39.770 INFO:teuthology.orchestra.run.trial006.stdout: ceph-fuse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:39.770 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:39.770 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:39.798 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install ceph-volume 2026-02-18T06:34:39.856 INFO:teuthology.orchestra.run.trial051.stdout:rbd-mirror-20.2.0-670.gadc19233.el9.x86_64.rpm 20 MB/s | 2.9 MB 00:00 2026-02-18T06:34:39.856 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:39.857 INFO:teuthology.orchestra.run.trial051.stdout:Total 20 MB/s | 2.9 MB 00:00 2026-02-18T06:34:39.857 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:39.861 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:39.862 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:39.901 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:39.901 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:40.083 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:40.084 INFO:teuthology.orchestra.run.trial051.stdout: Installing : rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.106 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:52 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:40.107 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.107 INFO:teuthology.orchestra.run.trial051.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:40.108 INFO:teuthology.orchestra.run.trial051.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-18T06:34:40.108 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T06:34:40.108 INFO:teuthology.orchestra.run.trial051.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T06:34:40.108 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:40.171 INFO:teuthology.orchestra.run.trial006.stdout:Package ceph-volume-2:20.2.0-670.gadc19233.el9.noarch is already installed. 2026-02-18T06:34:40.189 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:40.189 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:40.190 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:40.210 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install librados-devel 2026-02-18T06:34:40.425 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.425 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:40.425 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:40.426 INFO:teuthology.orchestra.run.trial051.stdout: rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:40.426 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:40.426 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:40.453 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install rbd-nbd 2026-02-18T06:34:40.523 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:52 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:40.604 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repo Size 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout: librados-devel x86_64 2:20.2.0-670.gadc19233.el9 ceph 126 k 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 126 k 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 449 k 2026-02-18T06:34:40.605 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:40.691 INFO:teuthology.orchestra.run.trial006.stdout:librados-devel-20.2.0-670.gadc19233.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-18T06:34:40.691 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:40.691 INFO:teuthology.orchestra.run.trial006.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-18T06:34:40.691 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:40.694 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:40.694 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:40.725 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:40.725 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:40.765 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:53 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:40.779 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:40.797 INFO:teuthology.orchestra.run.trial006.stdout: Installing : librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.846 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout:Installing: 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout: rbd-nbd x86_64 2:20.2.0-670.gadc19233.el9 ceph 180 k 2026-02-18T06:34:40.848 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:Transaction Summary 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:================================================================================ 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:Install 1 Package 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:Total download size: 180 k 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:Installed size: 498 k 2026-02-18T06:34:40.849 INFO:teuthology.orchestra.run.trial051.stdout:Downloading Packages: 2026-02-18T06:34:40.876 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.932 INFO:teuthology.orchestra.run.trial051.stdout:rbd-nbd-20.2.0-670.gadc19233.el9.x86_64.rpm 2.1 MB/s | 180 kB 00:00 2026-02-18T06:34:40.933 INFO:teuthology.orchestra.run.trial051.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:40.933 INFO:teuthology.orchestra.run.trial051.stdout:Total 2.0 MB/s | 180 kB 00:00 2026-02-18T06:34:40.933 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction check 2026-02-18T06:34:40.934 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:40.934 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:40.935 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:40.935 INFO:teuthology.orchestra.run.trial006.stdout: librados-devel-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:40.935 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:40.935 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:40.938 INFO:teuthology.orchestra.run.trial051.stdout:Transaction check succeeded. 2026-02-18T06:34:40.938 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction test 2026-02-18T06:34:40.963 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install libcephfs2 2026-02-18T06:34:40.971 INFO:teuthology.orchestra.run.trial051.stdout:Transaction test succeeded. 2026-02-18T06:34:40.972 INFO:teuthology.orchestra.run.trial051.stdout:Running transaction 2026-02-18T06:34:41.028 INFO:teuthology.orchestra.run.trial051.stdout: Preparing : 1/1 2026-02-18T06:34:41.045 INFO:teuthology.orchestra.run.trial051.stdout: Installing : rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.124 INFO:teuthology.orchestra.run.trial051.stdout: Running scriptlet: rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout: Verifying : rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout:Installed: 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout: rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:41.182 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:41.211 DEBUG:teuthology.orchestra.run.trial051:> sudo yum -y install rbd-mirror 2026-02-18T06:34:41.271 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:53 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:41.336 INFO:teuthology.orchestra.run.trial006.stdout:Package libcephfs2-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:41.354 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:41.354 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:41.355 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:41.376 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install libcephfs-devel 2026-02-18T06:34:41.532 INFO:teuthology.orchestra.run.trial051.stdout:Last metadata expiration check: 0:00:54 ago on Wed 18 Feb 2026 06:33:47 AM UTC. 2026-02-18T06:34:41.596 INFO:teuthology.orchestra.run.trial051.stdout:Package rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:41.614 INFO:teuthology.orchestra.run.trial051.stdout:Dependencies resolved. 2026-02-18T06:34:41.615 INFO:teuthology.orchestra.run.trial051.stdout:Nothing to do. 2026-02-18T06:34:41.615 INFO:teuthology.orchestra.run.trial051.stdout:Complete! 2026-02-18T06:34:41.638 DEBUG:teuthology.parallel:result is None 2026-02-18T06:34:41.688 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:53 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:41.768 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repo Size 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-devel x86_64 2:20.2.0-670.gadc19233.el9 ceph 34 k 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 34 k 2026-02-18T06:34:41.769 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 155 k 2026-02-18T06:34:41.770 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:41.831 INFO:teuthology.orchestra.run.trial006.stdout:libcephfs-devel-20.2.0-670.gadc19233.el9.x86_64 556 kB/s | 34 kB 00:00 2026-02-18T06:34:41.831 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:41.831 INFO:teuthology.orchestra.run.trial006.stdout:Total 550 kB/s | 34 kB 00:00 2026-02-18T06:34:41.831 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:41.834 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:41.835 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:41.836 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:41.836 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:41.857 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:41.873 INFO:teuthology.orchestra.run.trial006.stdout: Installing : libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.931 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout: libcephfs-devel-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:41.989 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:42.017 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install librados2 2026-02-18T06:34:42.327 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:54 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:42.391 INFO:teuthology.orchestra.run.trial006.stdout:Package librados2-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:42.408 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:42.409 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:42.409 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:42.431 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install librbd1 2026-02-18T06:34:42.742 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:54 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:42.807 INFO:teuthology.orchestra.run.trial006.stdout:Package librbd1-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:42.825 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:42.826 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:42.826 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:42.847 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install python3-rados 2026-02-18T06:34:43.157 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:55 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:43.222 INFO:teuthology.orchestra.run.trial006.stdout:Package python3-rados-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:43.240 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:43.241 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:43.241 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:43.263 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install python3-rgw 2026-02-18T06:34:43.569 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:55 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:43.633 INFO:teuthology.orchestra.run.trial006.stdout:Package python3-rgw-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:43.651 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:43.652 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:43.652 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:43.673 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install python3-cephfs 2026-02-18T06:34:43.986 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:55 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:44.051 INFO:teuthology.orchestra.run.trial006.stdout:Package python3-cephfs-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:44.068 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:44.069 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:44.069 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:44.090 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install python3-rbd 2026-02-18T06:34:44.402 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:56 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:44.465 INFO:teuthology.orchestra.run.trial006.stdout:Package python3-rbd-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:44.483 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:44.484 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:44.484 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:44.505 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install rbd-fuse 2026-02-18T06:34:44.816 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:56 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:44.899 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:44.899 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout: rbd-fuse x86_64 2:20.2.0-670.gadc19233.el9 ceph 91 k 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:44.900 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:44.901 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 91 k 2026-02-18T06:34:44.901 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 238 k 2026-02-18T06:34:44.901 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:44.973 INFO:teuthology.orchestra.run.trial006.stdout:rbd-fuse-20.2.0-670.gadc19233.el9.x86_64.rpm 1.2 MB/s | 91 kB 00:00 2026-02-18T06:34:44.973 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:44.973 INFO:teuthology.orchestra.run.trial006.stdout:Total 1.2 MB/s | 91 kB 00:00 2026-02-18T06:34:44.973 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:44.978 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:44.978 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:45.009 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:45.009 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:45.060 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:45.077 INFO:teuthology.orchestra.run.trial006.stdout: Installing : rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:45.160 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:45.217 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:45.217 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:45.217 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:45.218 INFO:teuthology.orchestra.run.trial006.stdout: rbd-fuse-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:45.218 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:45.218 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:45.246 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install rbd-mirror 2026-02-18T06:34:45.568 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:57 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:45.650 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout: Package Arch Version Repo Size 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout: rbd-mirror x86_64 2:20.2.0-670.gadc19233.el9 ceph 2.9 M 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:45.651 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:45.652 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:45.652 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 2.9 M 2026-02-18T06:34:45.652 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 11 M 2026-02-18T06:34:45.652 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:45.795 INFO:teuthology.orchestra.run.trial006.stdout:rbd-mirror-20.2.0-670.gadc19233.el9.x86_64.rpm 20 MB/s | 2.9 MB 00:00 2026-02-18T06:34:45.796 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:45.796 INFO:teuthology.orchestra.run.trial006.stdout:Total 20 MB/s | 2.9 MB 00:00 2026-02-18T06:34:45.796 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:45.801 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:45.801 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:45.843 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:45.843 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:46.032 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:46.033 INFO:teuthology.orchestra.run.trial006.stdout: Installing : rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:46.055 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:46.055 INFO:teuthology.orchestra.run.trial006.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-18T06:34:46.056 INFO:teuthology.orchestra.run.trial006.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-18T06:34:46.056 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T06:34:46.056 INFO:teuthology.orchestra.run.trial006.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-18T06:34:46.056 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:46.364 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:46.365 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:46.365 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:46.365 INFO:teuthology.orchestra.run.trial006.stdout: rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:46.365 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:46.365 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:46.392 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install rbd-nbd 2026-02-18T06:34:46.708 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:58 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:46.789 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout: Package Architecture Version Repository Size 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout:Installing: 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout: rbd-nbd x86_64 2:20.2.0-670.gadc19233.el9 ceph 180 k 2026-02-18T06:34:46.790 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:Transaction Summary 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:================================================================================ 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:Install 1 Package 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:Total download size: 180 k 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:Installed size: 498 k 2026-02-18T06:34:46.791 INFO:teuthology.orchestra.run.trial006.stdout:Downloading Packages: 2026-02-18T06:34:46.875 INFO:teuthology.orchestra.run.trial006.stdout:rbd-nbd-20.2.0-670.gadc19233.el9.x86_64.rpm 2.1 MB/s | 180 kB 00:00 2026-02-18T06:34:46.875 INFO:teuthology.orchestra.run.trial006.stdout:-------------------------------------------------------------------------------- 2026-02-18T06:34:46.875 INFO:teuthology.orchestra.run.trial006.stdout:Total 2.1 MB/s | 180 kB 00:00 2026-02-18T06:34:46.875 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction check 2026-02-18T06:34:46.880 INFO:teuthology.orchestra.run.trial006.stdout:Transaction check succeeded. 2026-02-18T06:34:46.880 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction test 2026-02-18T06:34:46.911 INFO:teuthology.orchestra.run.trial006.stdout:Transaction test succeeded. 2026-02-18T06:34:46.911 INFO:teuthology.orchestra.run.trial006.stdout:Running transaction 2026-02-18T06:34:46.967 INFO:teuthology.orchestra.run.trial006.stdout: Preparing : 1/1 2026-02-18T06:34:46.984 INFO:teuthology.orchestra.run.trial006.stdout: Installing : rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:47.067 INFO:teuthology.orchestra.run.trial006.stdout: Running scriptlet: rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout: Verifying : rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 1/1 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout:Installed: 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout: rbd-nbd-2:20.2.0-670.gadc19233.el9.x86_64 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:47.125 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:47.154 DEBUG:teuthology.orchestra.run.trial006:> sudo yum -y install rbd-mirror 2026-02-18T06:34:47.468 INFO:teuthology.orchestra.run.trial006.stdout:Last metadata expiration check: 0:00:59 ago on Wed 18 Feb 2026 06:33:48 AM UTC. 2026-02-18T06:34:47.532 INFO:teuthology.orchestra.run.trial006.stdout:Package rbd-mirror-2:20.2.0-670.gadc19233.el9.x86_64 is already installed. 2026-02-18T06:34:47.550 INFO:teuthology.orchestra.run.trial006.stdout:Dependencies resolved. 2026-02-18T06:34:47.551 INFO:teuthology.orchestra.run.trial006.stdout:Nothing to do. 2026-02-18T06:34:47.551 INFO:teuthology.orchestra.run.trial006.stdout:Complete! 2026-02-18T06:34:47.576 DEBUG:teuthology.parallel:result is None 2026-02-18T06:34:47.577 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:34:47.750 DEBUG:teuthology.orchestra.run.trial006:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-18T06:34:47.772 INFO:teuthology.orchestra.run.trial006.stdout:20.2.0-670.gadc19233.el9 2026-02-18T06:34:47.773 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-670.gadc19233.el9 2026-02-18T06:34:47.773 INFO:teuthology.task.install:The correct ceph version 20.2.0-670.gadc19233 is installed. 2026-02-18T06:34:47.776 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:34:47.906 DEBUG:teuthology.orchestra.run.trial051:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-18T06:34:47.927 INFO:teuthology.orchestra.run.trial051.stdout:20.2.0-670.gadc19233.el9 2026-02-18T06:34:47.927 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-670.gadc19233.el9 2026-02-18T06:34:47.928 INFO:teuthology.task.install:The correct ceph version 20.2.0-670.gadc19233 is installed. 2026-02-18T06:34:47.929 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-18T06:34:47.929 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:47.929 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-18T06:34:47.954 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:47.954 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-18T06:34:47.991 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-18T06:34:47.991 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:47.991 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/usr/bin/daemon-helper 2026-02-18T06:34:48.017 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-18T06:34:48.084 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:48.084 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/usr/bin/daemon-helper 2026-02-18T06:34:48.110 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-18T06:34:48.173 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-18T06:34:48.173 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:48.174 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-18T06:34:48.199 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-18T06:34:48.264 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:48.264 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-18T06:34:48.289 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-18T06:34:48.353 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-18T06:34:48.353 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:48.353 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/usr/bin/stdin-killer 2026-02-18T06:34:48.379 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-18T06:34:48.444 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:48.444 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/usr/bin/stdin-killer 2026-02-18T06:34:48.468 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-18T06:34:48.532 INFO:teuthology.run_tasks:Running task ceph... 2026-02-18T06:34:48.649 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-18T06:34:48.649 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod 777 /var/log/ceph 2026-02-18T06:34:48.651 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod 777 /var/log/ceph 2026-02-18T06:34:48.673 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-18T06:34:48.673 DEBUG:teuthology.orchestra.run.trial006:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-18T06:34:48.715 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-18T06:34:48.739 INFO:tasks.ceph:Creating extra log directories... 2026-02-18T06:34:48.739 DEBUG:teuthology.orchestra.run.trial006:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-18T06:34:48.778 DEBUG:teuthology.orchestra.run.trial051:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-18T06:34:48.823 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-18T06:34:48.823 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-18T06:34:48.823 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909', 'branch': '20.2.1_2.17.26-baseline_2', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '54909', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': '20.2.1_2.17.26-baseline_2'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': '20.2.1_2.17.26-baseline_2', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7878, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': '20.2.1_2.17.26-baseline_2', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'targets': {'trial006.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPP05jXG1JMiRsC5EKWKC4JiRMBDH/oEdWq9Zi4ru9TMCkTxM7/9hlIOEHzP+L3FV2XDq89oJpv4o0bMYCJe/V8=', 'trial051.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXWP8acdBulI9g3phySvn7Yy8qF7TW7SBOxFGID5vjvI+2GKGhwlNx0Ib+pGTj2SVsuhiqbtgtA24CXqUWxQak='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-17_21:26:40', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682'} 2026-02-18T06:34:48.824 DEBUG:teuthology.orchestra.run.trial006:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-18T06:34:48.843 DEBUG:teuthology.orchestra.run.trial051:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-18T06:34:48.859 DEBUG:teuthology.orchestra.run.trial006:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T06:34:48.899 DEBUG:teuthology.orchestra.run.trial051:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T06:34:48.923 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:48.923 DEBUG:teuthology.orchestra.run.trial006:> dd if=/scratch_devs of=/dev/stdout 2026-02-18T06:34:48.977 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-18T06:34:48.977 DEBUG:teuthology.orchestra.run.trial006:> stat /dev/vg_nvme/lv_1 2026-02-18T06:34:49.034 INFO:teuthology.orchestra.run.trial006.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-18T06:34:49.034 INFO:teuthology.orchestra.run.trial006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:34:49.034 INFO:teuthology.orchestra.run.trial006.stdout:Device: 6h/6d Inode: 1303 Links: 1 2026-02-18T06:34:49.034 INFO:teuthology.orchestra.run.trial006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:34:49.035 INFO:teuthology.orchestra.run.trial006.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:34:49.035 INFO:teuthology.orchestra.run.trial006.stdout:Access: 2026-02-18 06:34:46.192156878 +0000 2026-02-18T06:34:49.035 INFO:teuthology.orchestra.run.trial006.stdout:Modify: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.035 INFO:teuthology.orchestra.run.trial006.stdout:Change: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.035 INFO:teuthology.orchestra.run.trial006.stdout: Birth: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.035 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-18T06:34:49.098 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records in 2026-02-18T06:34:49.098 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records out 2026-02-18T06:34:49.098 INFO:teuthology.orchestra.run.trial006.stderr:512 bytes copied, 0.000164339 s, 3.1 MB/s 2026-02-18T06:34:49.099 DEBUG:teuthology.orchestra.run.trial006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-18T06:34:49.156 DEBUG:teuthology.orchestra.run.trial006:> stat /dev/vg_nvme/lv_2 2026-02-18T06:34:49.215 INFO:teuthology.orchestra.run.trial006.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-18T06:34:49.215 INFO:teuthology.orchestra.run.trial006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:34:49.215 INFO:teuthology.orchestra.run.trial006.stdout:Device: 6h/6d Inode: 1309 Links: 1 2026-02-18T06:34:49.215 INFO:teuthology.orchestra.run.trial006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:34:49.215 INFO:teuthology.orchestra.run.trial006.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:34:49.216 INFO:teuthology.orchestra.run.trial006.stdout:Access: 2026-02-18 06:34:46.192156878 +0000 2026-02-18T06:34:49.216 INFO:teuthology.orchestra.run.trial006.stdout:Modify: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.216 INFO:teuthology.orchestra.run.trial006.stdout:Change: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.216 INFO:teuthology.orchestra.run.trial006.stdout: Birth: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.216 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-18T06:34:49.277 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records in 2026-02-18T06:34:49.278 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records out 2026-02-18T06:34:49.278 INFO:teuthology.orchestra.run.trial006.stderr:512 bytes copied, 0.00014332 s, 3.6 MB/s 2026-02-18T06:34:49.279 DEBUG:teuthology.orchestra.run.trial006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-18T06:34:49.337 DEBUG:teuthology.orchestra.run.trial006:> stat /dev/vg_nvme/lv_3 2026-02-18T06:34:49.394 INFO:teuthology.orchestra.run.trial006.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Device: 6h/6d Inode: 1295 Links: 1 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Access: 2026-02-18 06:34:46.193156853 +0000 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Modify: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout:Change: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.395 INFO:teuthology.orchestra.run.trial006.stdout: Birth: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.396 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-18T06:34:49.458 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records in 2026-02-18T06:34:49.458 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records out 2026-02-18T06:34:49.458 INFO:teuthology.orchestra.run.trial006.stderr:512 bytes copied, 0.000158889 s, 3.2 MB/s 2026-02-18T06:34:49.459 DEBUG:teuthology.orchestra.run.trial006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-18T06:34:49.517 DEBUG:teuthology.orchestra.run.trial006:> stat /dev/vg_nvme/lv_4 2026-02-18T06:34:49.574 INFO:teuthology.orchestra.run.trial006.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-18T06:34:49.574 INFO:teuthology.orchestra.run.trial006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:34:49.574 INFO:teuthology.orchestra.run.trial006.stdout:Device: 6h/6d Inode: 1301 Links: 1 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout:Access: 2026-02-18 06:34:46.193156853 +0000 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout:Modify: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout:Change: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.575 INFO:teuthology.orchestra.run.trial006.stdout: Birth: 2026-02-18 06:33:55.491399249 +0000 2026-02-18T06:34:49.575 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-18T06:34:49.638 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records in 2026-02-18T06:34:49.639 INFO:teuthology.orchestra.run.trial006.stderr:1+0 records out 2026-02-18T06:34:49.639 INFO:teuthology.orchestra.run.trial006.stderr:512 bytes copied, 0.000144453 s, 3.5 MB/s 2026-02-18T06:34:49.640 DEBUG:teuthology.orchestra.run.trial006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-18T06:34:49.697 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:34:49.697 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial006.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T06:34:49.697 INFO:tasks.ceph:Generating config... 2026-02-18T06:34:49.700 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-18T06:34:49.700 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-18T06:34:49.700 INFO:tasks.ceph:[client] log to stderr = False 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:49.701 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:49.701 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-18T06:34:49.702 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-18T06:34:49.702 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-18T06:34:49.702 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-18T06:34:49.702 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-18T06:34:49.702 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-18T06:34:49.702 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-18T06:34:49.702 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-18T06:34:49.702 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-18T06:34:49.702 INFO:tasks.ceph:[osd] bluestore compression algorithm = zlib 2026-02-18T06:34:49.702 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-18T06:34:49.702 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-18T06:34:49.702 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-18T06:34:49.703 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-18T06:34:49.703 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-18T06:34:49.704 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-18T06:34:49.773 INFO:teuthology.orchestra.run.trial006.stdout:creating /etc/ceph/cluster1.keyring 2026-02-18T06:34:49.776 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-18T06:34:49.853 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T06:34:49.918 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.6')] 2026-02-18T06:34:49.919 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.6', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-18T06:34:49.921 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:49.921 DEBUG:teuthology.orchestra.run.trial006:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T06:34:49.974 DEBUG:teuthology.orchestra.run.trial006:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.6 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:50.047 INFO:teuthology.orchestra.run.trial006.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:monmaptool: generated fsid 8af030b1-7761-4bce-8b14-c3dac9d5a86a 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:setting min_mon_release = tentacle 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:epoch 0 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:fsid 8af030b1-7761-4bce-8b14-c3dac9d5a86a 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:last_changed 2026-02-18T06:34:50.046424+0000 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:created 2026-02-18T06:34:50.046424+0000 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:min_mon_release 20 (tentacle) 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:election_strategy: 1 2026-02-18T06:34:50.048 INFO:teuthology.orchestra.run.trial006.stdout:0: [v2:10.20.193.6:3300/0,v1:10.20.193.6:6789/0] mon.a 2026-02-18T06:34:50.049 INFO:teuthology.orchestra.run.trial006.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-18T06:34:50.050 DEBUG:teuthology.orchestra.run.trial006:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T06:34:50.105 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 8af030b1-7761-4bce-8b14-c3dac9d5a86a... 2026-02-18T06:34:50.105 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-18T06:34:50.147 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-18T06:34:50.184 INFO:teuthology.orchestra.run.trial006.stdout:[global] 2026-02-18T06:34:50.184 INFO:teuthology.orchestra.run.trial006.stdout: chdir = "" 2026-02-18T06:34:50.184 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T06:34:50.184 INFO:teuthology.orchestra.run.trial006.stdout: auth supported = cephx 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: filestore xattr use omap = true 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: mon clock drift allowed = 1.000 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: osd crush chooseleaf type = 0 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: auth debug = true 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: ms die on old message = true 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: ms die on bug = true 2026-02-18T06:34:50.185 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon pg warn max object skew = 0 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: osd pool default size = 2 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon osd allow primary affinity = true 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon osd allow pg remap = true 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on legacy crush tunables = false 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on crush straw calc version zero = false 2026-02-18T06:34:50.186 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on no sortbitwise = false 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on osd down out interval zero = false 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on too few osds = false 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon_allow_pool_size_one = true 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: osd default data pool replay window = 5 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: mon allow pool delete = true 2026-02-18T06:34:50.187 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: mon cluster log file level = debug 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: debug asserts on shutdown = true 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: mon health detail to clog = false 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: mon host = 10.20.193.6 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: mon client directed command retry = 5 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on pool no app = False 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: ms inject socket failures = 5000 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: fsid = 8af030b1-7761-4bce-8b14-c3dac9d5a86a 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout:[osd] 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: osd journal size = 100 2026-02-18T06:34:50.188 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd scrub load threshold = 5.0 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd scrub max interval = 600 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd mclock profile = high_recovery_ops 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd mclock skip benchmark = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd recover clone overlap = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd recovery max chunk = 1048576 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd debug shutdown = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd debug op order = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd debug verify stray on activate = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: osd debug trim objects = true 2026-02-18T06:34:50.189 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: osd open classes on start = true 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: osd debug pg log writeout = true 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: osd deep scrub update digest min age = 30 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: osd map max advance = 10 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: journal zero on create = true 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: filestore ondisk finisher threads = 3 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: filestore apply finisher threads = 3 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.190 INFO:teuthology.orchestra.run.trial006.stdout: bdev debug aio = true 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: osd debug misdirected ops = true 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: bluestore block size = 96636764160 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: bluestore compression algorithm = zlib 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: bluestore compression mode = aggressive 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: bluestore fsck on mount = True 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: debug bluefs = 1/20 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: debug bluestore = 1/20 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: debug osd = 20 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: debug rocksdb = 4/10 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: mon osd full ratio = 0.9 2026-02-18T06:34:50.191 INFO:teuthology.orchestra.run.trial006.stdout: mon osd nearfull ratio = 0.8 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: osd failsafe full ratio = 0.95 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: osd objectstore = bluestore 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout:[mgr] 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: debug mgr = 20 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: debug mon = 20 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: debug auth = 20 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: mgr/telemetry/nag = false 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.192 INFO:teuthology.orchestra.run.trial006.stdout:[mon] 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: debug mon = 20 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: debug paxos = 20 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: debug auth = 20 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon data avail warn = 5 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon mgr mkfs grace = 240 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon osd reporter subtree level = osd 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon osd prime pg temp = true 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.193 INFO:teuthology.orchestra.run.trial006.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: # 1m isn't quite enough 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: mon_down_mkfs_grace = 2m 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_filestore_osds = false 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:50.194 INFO:teuthology.orchestra.run.trial006.stdout:[client] 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: rgw cache enabled = true 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: rgw enable ops log = true 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: rgw enable usage log = true 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: debug rbd = 20 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: debug rbd_mirror = 15 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: log to stderr = False 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: rbd default features = 125 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: rbd mirror image policy type = simple 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout:[mon.a] 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.0] 2026-02-18T06:34:50.195 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.1] 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.2] 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.3] 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.196 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.198 INFO:teuthology.orchestra.run.trial051.stdout:[global] 2026-02-18T06:34:50.198 INFO:teuthology.orchestra.run.trial051.stdout: chdir = "" 2026-02-18T06:34:50.198 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T06:34:50.198 INFO:teuthology.orchestra.run.trial051.stdout: auth supported = cephx 2026-02-18T06:34:50.198 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: filestore xattr use omap = true 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: mon clock drift allowed = 1.000 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: osd crush chooseleaf type = 0 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: auth debug = true 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: ms die on old message = true 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: ms die on bug = true 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.199 INFO:teuthology.orchestra.run.trial051.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon pg warn max object skew = 0 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: osd pool default size = 2 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon osd allow primary affinity = true 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon osd allow pg remap = true 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on legacy crush tunables = false 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on crush straw calc version zero = false 2026-02-18T06:34:50.200 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on no sortbitwise = false 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on osd down out interval zero = false 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on too few osds = false 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon_allow_pool_size_one = true 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: osd default data pool replay window = 5 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon allow pool delete = true 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.201 INFO:teuthology.orchestra.run.trial051.stdout: mon cluster log file level = debug 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: debug asserts on shutdown = true 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: mon health detail to clog = false 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: mon host = 10.20.193.6 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: mon client directed command retry = 5 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on pool no app = False 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: ms inject socket failures = 5000 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: fsid = 8af030b1-7761-4bce-8b14-c3dac9d5a86a 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout:[osd] 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: osd journal size = 100 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.202 INFO:teuthology.orchestra.run.trial051.stdout: osd scrub load threshold = 5.0 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd scrub max interval = 600 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd mclock profile = high_recovery_ops 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd mclock skip benchmark = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd recover clone overlap = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd recovery max chunk = 1048576 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd debug shutdown = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd debug op order = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd debug verify stray on activate = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd debug trim objects = true 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.203 INFO:teuthology.orchestra.run.trial051.stdout: osd open classes on start = true 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: osd debug pg log writeout = true 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: osd deep scrub update digest min age = 30 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: osd map max advance = 10 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: journal zero on create = true 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: filestore ondisk finisher threads = 3 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: filestore apply finisher threads = 3 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.204 INFO:teuthology.orchestra.run.trial051.stdout: bdev debug aio = true 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: osd debug misdirected ops = true 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: bluestore block size = 96636764160 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: bluestore compression algorithm = zlib 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: bluestore compression mode = aggressive 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: bluestore fsck on mount = True 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: debug bluefs = 1/20 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: debug bluestore = 1/20 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: debug osd = 20 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: debug rocksdb = 4/10 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: mon osd full ratio = 0.9 2026-02-18T06:34:50.205 INFO:teuthology.orchestra.run.trial051.stdout: mon osd nearfull ratio = 0.8 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: osd failsafe full ratio = 0.95 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: osd objectstore = bluestore 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout:[mgr] 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: debug mgr = 20 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: debug mon = 20 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: debug auth = 20 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: mgr/telemetry/nag = false 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.206 INFO:teuthology.orchestra.run.trial051.stdout:[mon] 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: debug mon = 20 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: debug paxos = 20 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: debug auth = 20 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon data avail warn = 5 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon mgr mkfs grace = 240 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon osd reporter subtree level = osd 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon osd prime pg temp = true 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.207 INFO:teuthology.orchestra.run.trial051.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: # 1m isn't quite enough 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: mon_down_mkfs_grace = 2m 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_filestore_osds = false 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:34:50.208 INFO:teuthology.orchestra.run.trial051.stdout:[client] 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: rgw cache enabled = true 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: rgw enable ops log = true 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: rgw enable usage log = true 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: debug rbd = 20 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: debug rbd_mirror = 15 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: log to stderr = False 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: rbd default features = 125 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: rbd mirror image policy type = simple 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout:[mon.a] 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.0] 2026-02-18T06:34:50.209 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.1] 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.2] 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.3] 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:34:50.210 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:34:50.211 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-18T06:34:50.211 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-18T06:34:50.281 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-18T06:34:50.281 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:50.281 DEBUG:teuthology.orchestra.run.trial006:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-18T06:34:50.337 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:50.337 DEBUG:teuthology.orchestra.run.trial006:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-18T06:34:50.393 INFO:tasks.ceph:Sending monmap to node ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:34:50.393 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:50.393 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-18T06:34:50.393 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T06:34:50.465 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:50.466 DEBUG:teuthology.orchestra.run.trial006:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:50.522 INFO:tasks.ceph:Sending monmap to node ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:34:50.522 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:50.522 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-18T06:34:50.522 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-18T06:34:50.557 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:50.557 DEBUG:teuthology.orchestra.run.trial051:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:50.615 INFO:tasks.ceph:Setting up mon nodes... 2026-02-18T06:34:50.616 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-18T06:34:50.616 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-18T06:34:50.665 INFO:teuthology.orchestra.run.trial006.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-18T06:34:50.668 INFO:tasks.ceph:Setting up mds nodes... 2026-02-18T06:34:50.668 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-18T06:34:50.669 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-18T06:34:50.749 INFO:teuthology.orchestra.run.trial006.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-18T06:34:50.759 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-18T06:34:50.796 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-18T06:34:50.807 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-18T06:34:50.881 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-18T06:34:50.892 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-18T06:34:50.966 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-18T06:34:50.976 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-18T06:34:51.050 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-18T06:34:51.061 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-18T06:34:51.134 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-18T06:34:51.145 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-18T06:34:51.220 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-18T06:34:51.232 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-18T06:34:51.304 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-18T06:34:51.315 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-18T06:34:51.388 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-18T06:34:51.398 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-18T06:34:51.398 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial006.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T06:34:51.399 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-18T06:34:51.424 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:34:51.424 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-18T06:34:51.424 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:34:51.424 DEBUG:teuthology.orchestra.run.trial006:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-18T06:34:51.489 INFO:teuthology.orchestra.run.trial006.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:34:51.490 INFO:teuthology.orchestra.run.trial006.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:34:52.087 INFO:teuthology.orchestra.run.trial006.stdout:Discarding blocks...Done. 2026-02-18T06:34:52.089 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial006.front.sepia.ceph.com -o noatime 2026-02-18T06:34:52.090 DEBUG:teuthology.orchestra.run.trial006:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-18T06:34:52.158 DEBUG:teuthology.orchestra.run.trial006:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-18T06:34:52.225 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-18T06:34:52.289 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:34:52.289 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-18T06:34:52.289 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:34:52.290 DEBUG:teuthology.orchestra.run.trial006:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-18T06:34:52.354 INFO:teuthology.orchestra.run.trial006.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:34:52.355 INFO:teuthology.orchestra.run.trial006.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:34:52.913 INFO:teuthology.orchestra.run.trial006.stdout:Discarding blocks...Done. 2026-02-18T06:34:52.915 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial006.front.sepia.ceph.com -o noatime 2026-02-18T06:34:52.915 DEBUG:teuthology.orchestra.run.trial006:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-18T06:34:52.983 DEBUG:teuthology.orchestra.run.trial006:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-18T06:34:53.049 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-18T06:34:53.115 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:34:53.115 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-18T06:34:53.115 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:34:53.115 DEBUG:teuthology.orchestra.run.trial006:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-18T06:34:53.179 INFO:teuthology.orchestra.run.trial006.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:34:53.179 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:34:53.179 INFO:teuthology.orchestra.run.trial006.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:34:53.180 INFO:teuthology.orchestra.run.trial006.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:34:53.750 INFO:teuthology.orchestra.run.trial006.stdout:Discarding blocks...Done. 2026-02-18T06:34:53.751 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial006.front.sepia.ceph.com -o noatime 2026-02-18T06:34:53.751 DEBUG:teuthology.orchestra.run.trial006:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-18T06:34:53.822 DEBUG:teuthology.orchestra.run.trial006:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-18T06:34:53.889 DEBUG:teuthology.orchestra.run.trial006:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:53.968 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:53.967+0000 7fc04e817900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-18T06:34:53.969 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:53.967+0000 7fc04e817900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-18T06:34:53.969 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:53.967+0000 7fc04e817900 -1 bdev(0x55e494f57800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-18T06:34:53.969 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:53.967+0000 7fc04e817900 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-18T06:34:54.422 DEBUG:teuthology.orchestra.run.trial006:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-18T06:34:54.487 DEBUG:teuthology.orchestra.run.trial006:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:54.568 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:54.566+0000 7fc2ab0fd900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-18T06:34:54.568 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:54.567+0000 7fc2ab0fd900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-18T06:34:54.569 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:54.567+0000 7fc2ab0fd900 -1 bdev(0x55f059473800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-18T06:34:54.569 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:54.567+0000 7fc2ab0fd900 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-18T06:34:55.029 DEBUG:teuthology.orchestra.run.trial006:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-18T06:34:55.094 DEBUG:teuthology.orchestra.run.trial006:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:55.173 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:55.171+0000 7ff1b53e0900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-18T06:34:55.174 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:55.172+0000 7ff1b53e0900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-18T06:34:55.174 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:55.172+0000 7ff1b53e0900 -1 bdev(0x55fe4230f800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-18T06:34:55.174 INFO:teuthology.orchestra.run.trial006.stderr:2026-02-18T06:34:55.172+0000 7ff1b53e0900 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-18T06:34:55.626 DEBUG:teuthology.orchestra.run.trial006:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-18T06:34:55.691 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-18T06:34:55.692 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:55.692 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-18T06:34:55.755 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:55.755 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-18T06:34:55.819 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:55.819 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-18T06:34:55.884 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:55.884 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-18T06:34:55.948 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:55.949 DEBUG:teuthology.orchestra.run.trial006:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-18T06:34:56.006 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.006 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-18T06:34:56.024 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.024 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-18T06:34:56.082 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.083 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-18T06:34:56.138 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.139 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-18T06:34:56.194 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.195 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-18T06:34:56.250 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.251 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-18T06:34:56.307 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.307 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-18T06:34:56.362 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:34:56.363 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-18T06:34:56.419 INFO:tasks.ceph:Adding keys to all mons... 2026-02-18T06:34:56.419 DEBUG:teuthology.orchestra.run.trial006:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-18T06:34:56.443 INFO:teuthology.orchestra.run.trial006.stdout:[mgr.x] 2026-02-18T06:34:56.443 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCKXZVpvqWYJxAAsPty3os/5jrFwxFLHKp/uA== 2026-02-18T06:34:56.443 INFO:teuthology.orchestra.run.trial006.stdout:[osd.0] 2026-02-18T06:34:56.443 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCNXZVpp0+xORAAEFxxI0gNRz7A7hDUBlaRZg== 2026-02-18T06:34:56.443 INFO:teuthology.orchestra.run.trial006.stdout:[osd.1] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCOXZVpMcfQIRAADUWE/U0lXGlBQmRDezDTBQ== 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout:[osd.2] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCPXZVp8WdEChAAtjLoxM7KBRyqMEyCAsoMZA== 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout:[client.0] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCKXZVpQWeULBAATXY6QJc/VmAwUP4NQHZkeA== 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCKXZVpofRdLxAAMfMCb1jusw4Rmzbjq+tHTA== 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.0] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCKXZVpOGR1NBAAKFpW8+YScStjSxOxmgH2DA== 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.1] 2026-02-18T06:34:56.444 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCKXZVpx1iDORAAhzF085iSzdJ8KuqK+NsWoA== 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.2] 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCLXZVph1f1AhAAFUIxmhCwAZfQr/nC2s3F6Q== 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.3] 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCLXZVp2ODzBxAA+WpUM+cgvq0gFI3u3J/25w== 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.4] 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCLXZVplpgVDRAAjAR1UhXItMMxgJLfgzALsQ== 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.5] 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCLXZVpEFIZEhAAOtbv+j9hz5BmXzWAIRC8Yw== 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.6] 2026-02-18T06:34:56.445 INFO:teuthology.orchestra.run.trial006.stdout: key = AQCLXZVp8FUQFxAAk+fKEwAX1S5hWavGAfv6tg== 2026-02-18T06:34:56.447 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-18T06:34:56.527 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:34:56.608 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:34:56.688 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:34:56.768 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:56.849 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:56.930 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.011 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.094 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.176 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.257 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.337 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.417 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:34:57.500 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-18T06:34:57.500 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-18T06:34:57.563 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-18T06:34:57.649 DEBUG:teuthology.orchestra.run.trial006:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-18T06:34:57.674 DEBUG:teuthology.orchestra.run.trial006:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-18T06:34:57.729 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-18T06:34:57.730 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-18T06:34:57.730 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-18T06:34:57.772 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-18T06:34:57.772 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-18T06:34:57.772 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-18T06:34:57.773 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-18T06:34:57.775 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-18T06:34:57.775 DEBUG:tasks.ceph:set 0 configs 2026-02-18T06:34:57.776 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 config dump 2026-02-18T06:34:57.876 INFO:teuthology.orchestra.run.trial006.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-18T06:34:57.883 INFO:tasks.ceph:Setting crush tunables to default 2026-02-18T06:34:57.884 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-18T06:34:57.984 INFO:teuthology.orchestra.run.trial006.stderr:adjusted tunables profile to default 2026-02-18T06:34:57.992 INFO:tasks.ceph:check_enable_crimson: False 2026-02-18T06:34:57.993 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-18T06:34:57.993 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:57.993 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-18T06:34:58.018 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:58.018 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-18T06:34:58.081 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:34:58.081 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-18T06:34:58.143 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 osd new 4406057c-7381-449c-a6a8-0e45373e3429 0 2026-02-18T06:34:58.285 INFO:teuthology.orchestra.run.trial006.stdout:0 2026-02-18T06:34:58.294 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 osd new f61faa8c-808b-4525-8a50-f48f1ff0ed96 1 2026-02-18T06:34:58.397 INFO:teuthology.orchestra.run.trial006.stdout:1 2026-02-18T06:34:58.405 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 osd new 53a83bfa-9f75-4cd4-937c-4464633a9dc4 2 2026-02-18T06:34:58.509 INFO:teuthology.orchestra.run.trial006.stdout:2 2026-02-18T06:34:58.517 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-18T06:34:58.517 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-18T06:34:58.518 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-18T06:34:58.518 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-18T06:34:58.518 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-18T06:34:58.520 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-18T06:34:58.520 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-18T06:34:58.520 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-18T06:34:58.521 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-18T06:34:58.521 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T06:34:58.626 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:34:58.626 INFO:teuthology.orchestra.run.trial006.stdout:{"epoch":5,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T06:34:58.508788+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:34:58.632 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-18T06:34:58.632 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-18T06:34:58.817 INFO:tasks.ceph.cluster1.osd.2.trial006.stderr:2026-02-18T06:34:58.816+0000 7f20cf80d900 -1 Falling back to public interface 2026-02-18T06:34:58.822 INFO:tasks.ceph.cluster1.osd.1.trial006.stderr:2026-02-18T06:34:58.821+0000 7fadf9210900 -1 Falling back to public interface 2026-02-18T06:34:58.822 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T06:34:58.821+0000 7f842b922900 -1 Falling back to public interface 2026-02-18T06:34:58.882 INFO:tasks.ceph.cluster1.mgr.x.trial006.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-18T06:34:58.882 INFO:tasks.ceph.cluster1.mgr.x.trial006.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-18T06:34:58.883 INFO:tasks.ceph.cluster1.mgr.x.trial006.stderr: from numpy import show_config as show_numpy_config 2026-02-18T06:34:58.934 DEBUG:teuthology.orchestra.run.trial006:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-18T06:34:59.229 INFO:teuthology.misc.health.trial006.stdout: 2026-02-18T06:34:59.230 INFO:teuthology.misc.health.trial006.stdout:{"epoch":5,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T06:34:58.508788+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:34:59.239 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-18T06:34:59.426 INFO:tasks.ceph.cluster1.osd.2.trial006.stderr:2026-02-18T06:34:59.425+0000 7f20cf80d900 -1 osd.2 0 log_to_monitors true 2026-02-18T06:34:59.440 INFO:tasks.ceph.cluster1.osd.1.trial006.stderr:2026-02-18T06:34:59.439+0000 7fadf9210900 -1 osd.1 0 log_to_monitors true 2026-02-18T06:34:59.457 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T06:34:59.456+0000 7f842b922900 -1 osd.0 0 log_to_monitors true 2026-02-18T06:35:00.843 INFO:tasks.ceph.cluster1.osd.2.trial006.stderr:2026-02-18T06:35:00.842+0000 7f20caf74640 -1 osd.2 0 waiting for initial osdmap 2026-02-18T06:35:00.844 INFO:tasks.ceph.cluster1.osd.1.trial006.stderr:2026-02-18T06:35:00.843+0000 7fadf489b640 -1 osd.1 0 waiting for initial osdmap 2026-02-18T06:35:00.844 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T06:35:00.843+0000 7f84278b0640 -1 osd.0 0 waiting for initial osdmap 2026-02-18T06:35:00.853 INFO:tasks.ceph.cluster1.osd.2.trial006.stderr:2026-02-18T06:35:00.852+0000 7f20c2572640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:00.857 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T06:35:00.856+0000 7f841eeae640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:00.857 INFO:tasks.ceph.cluster1.osd.1.trial006.stderr:2026-02-18T06:35:00.856+0000 7fadebe99640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:01.285 INFO:tasks.ceph.cluster1.mgr.x.trial006.stderr:2026-02-18T06:35:01.284+0000 7ff31b1df640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-18T06:35:05.542 DEBUG:teuthology.orchestra.run.trial006:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-18T06:35:05.715 INFO:teuthology.misc.health.trial006.stdout: 2026-02-18T06:35:05.715 INFO:teuthology.misc.health.trial006.stdout:{"epoch":11,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T06:35:05.286120+0000","last_up_change":"2026-02-18T06:35:01.841544+0000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:02.294274+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6809","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6811","nonce":660097312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6813","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6815","nonce":660097312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6821","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6823","nonce":660097312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6817","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6819","nonce":660097312}]},"public_addr":"10.20.193.6:6811/660097312","cluster_addr":"10.20.193.6:6815/660097312","heartbeat_back_addr":"10.20.193.6:6823/660097312","heartbeat_front_addr":"10.20.193.6:6819/660097312","state":["exists","up"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6808","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6810","nonce":1636389907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6812","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6814","nonce":1636389907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6820","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6822","nonce":1636389907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6816","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6818","nonce":1636389907}]},"public_addr":"10.20.193.6:6810/1636389907","cluster_addr":"10.20.193.6:6814/1636389907","heartbeat_back_addr":"10.20.193.6:6822/1636389907","heartbeat_front_addr":"10.20.193.6:6818/1636389907","state":["exists","up"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6800","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6801","nonce":1693096825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6802","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6803","nonce":1693096825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6806","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6807","nonce":1693096825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6804","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6805","nonce":1693096825}]},"public_addr":"10.20.193.6:6801/1693096825","cluster_addr":"10.20.193.6:6803/1693096825","heartbeat_back_addr":"10.20.193.6:6807/1693096825","heartbeat_front_addr":"10.20.193.6:6805/1693096825","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:05.723 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-18T06:35:05.723 INFO:tasks.ceph:Creating RBD pool 2026-02-18T06:35:05.723 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-18T06:35:06.292 INFO:teuthology.orchestra.run.trial006.stderr:pool 'rbd' created 2026-02-18T06:35:06.302 DEBUG:teuthology.orchestra.run.trial006:> rbd --cluster cluster1 pool init rbd 2026-02-18T06:35:09.307 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-18T06:35:09.308 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-18T06:35:09.308 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-18T06:35:09.490 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:09.499 INFO:teuthology.orchestra.run.trial006.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-18T06:35:09.500 INFO:tasks.ceph_manager:config epoch is 1 2026-02-18T06:35:09.500 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-18T06:35:09.500 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-18T06:35:09.500 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-18T06:35:09.710 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:09.718 INFO:teuthology.orchestra.run.trial006.stdout:{"epoch":5,"flags":0,"active_gid":4105,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6824","nonce":2757334562},{"type":"v1","addr":"10.20.193.6:6825","nonce":2757334562}]},"active_addr":"10.20.193.6:6825/2757334562","active_change":"2026-02-18T06:35:00.279009+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":1568220773}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":1815597778}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":3842354747}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":1508838452}]}]} 2026-02-18T06:35:09.719 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-18T06:35:09.719 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-18T06:35:09.719 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T06:35:09.899 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:09.899 INFO:teuthology.orchestra.run.trial006.stdout:{"epoch":15,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T06:35:09.296313+0000","last_up_change":"2026-02-18T06:35:01.841544+0000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:02.294274+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:05.901344+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6809","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6811","nonce":660097312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6813","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6815","nonce":660097312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6821","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6823","nonce":660097312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6817","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6819","nonce":660097312}]},"public_addr":"10.20.193.6:6811/660097312","cluster_addr":"10.20.193.6:6815/660097312","heartbeat_back_addr":"10.20.193.6:6823/660097312","heartbeat_front_addr":"10.20.193.6:6819/660097312","state":["exists","up"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6808","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6810","nonce":1636389907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6812","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6814","nonce":1636389907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6820","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6822","nonce":1636389907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6816","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6818","nonce":1636389907}]},"public_addr":"10.20.193.6:6810/1636389907","cluster_addr":"10.20.193.6:6814/1636389907","heartbeat_back_addr":"10.20.193.6:6822/1636389907","heartbeat_front_addr":"10.20.193.6:6818/1636389907","state":["exists","up"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6800","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6801","nonce":1693096825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6802","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6803","nonce":1693096825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6806","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6807","nonce":1693096825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6804","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6805","nonce":1693096825}]},"public_addr":"10.20.193.6:6801/1693096825","cluster_addr":"10.20.193.6:6803/1693096825","heartbeat_back_addr":"10.20.193.6:6807/1693096825","heartbeat_front_addr":"10.20.193.6:6805/1693096825","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:09.909 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-18T06:35:09.909 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T06:35:10.089 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:10.090 INFO:teuthology.orchestra.run.trial006.stdout:{"epoch":15,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T06:35:09.296313+0000","last_up_change":"2026-02-18T06:35:01.841544+0000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:02.294274+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:05.901344+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6809","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6811","nonce":660097312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6813","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6815","nonce":660097312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6821","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6823","nonce":660097312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6817","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6819","nonce":660097312}]},"public_addr":"10.20.193.6:6811/660097312","cluster_addr":"10.20.193.6:6815/660097312","heartbeat_back_addr":"10.20.193.6:6823/660097312","heartbeat_front_addr":"10.20.193.6:6819/660097312","state":["exists","up"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6808","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6810","nonce":1636389907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6812","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6814","nonce":1636389907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6820","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6822","nonce":1636389907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6816","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6818","nonce":1636389907}]},"public_addr":"10.20.193.6:6810/1636389907","cluster_addr":"10.20.193.6:6814/1636389907","heartbeat_back_addr":"10.20.193.6:6822/1636389907","heartbeat_front_addr":"10.20.193.6:6818/1636389907","state":["exists","up"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6800","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6801","nonce":1693096825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6802","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6803","nonce":1693096825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6806","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6807","nonce":1693096825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6804","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6805","nonce":1693096825}]},"public_addr":"10.20.193.6:6801/1693096825","cluster_addr":"10.20.193.6:6803/1693096825","heartbeat_back_addr":"10.20.193.6:6807/1693096825","heartbeat_front_addr":"10.20.193.6:6805/1693096825","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:10.100 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-18T06:35:10.100 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-18T06:35:10.101 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-18T06:35:10.191 INFO:teuthology.orchestra.run.trial006.stdout:34359738371 2026-02-18T06:35:10.191 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-18T06:35:10.192 INFO:teuthology.orchestra.run.trial006.stdout:34359738372 2026-02-18T06:35:10.193 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-18T06:35:10.194 INFO:teuthology.orchestra.run.trial006.stdout:34359738372 2026-02-18T06:35:10.194 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-18T06:35:10.414 INFO:teuthology.orchestra.run.trial006.stdout:34359738372 2026-02-18T06:35:10.417 INFO:teuthology.orchestra.run.trial006.stdout:34359738371 2026-02-18T06:35:10.423 INFO:teuthology.orchestra.run.trial006.stdout:34359738372 2026-02-18T06:35:10.426 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.1 2026-02-18T06:35:10.426 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:10.429 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.2 2026-02-18T06:35:10.430 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:10.434 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.0 2026-02-18T06:35:10.435 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:10.435 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-18T06:35:10.435 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T06:35:10.650 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:10.651 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T06:35:10.658 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-18T06:35:10.283088+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81384,"kb_used_data":504,"kb_used_omap":25,"kb_used_meta":80806,"kb_avail":283034136,"statfs":{"total":289910292480,"available":289826955264,"internally_reserved":0,"allocated":516096,"data_stored":1295371,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":25674,"internal_metadata":82746294},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.990622"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301780+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T06:35:09.301780+0000","last_peered":"2026-02-18T06:35:09.301780+0000","last_clean":"2026-02-18T06:35:09.301780+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T06:35:09.301780+0000","last_undegraded":"2026-02-18T06:35:09.301780+0000","last_fullsized":"2026-02-18T06:35:09.301780+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301102+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T06:35:09.301102+0000","last_peered":"2026-02-18T06:35:09.301102+0000","last_clean":"2026-02-18T06:35:09.301102+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T06:35:09.301102+0000","last_undegraded":"2026-02-18T06:35:09.301102+0000","last_fullsized":"2026-02-18T06:35:09.301102+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301717+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T06:35:09.301717+0000","last_peered":"2026-02-18T06:35:09.301717+0000","last_clean":"2026-02-18T06:35:09.301717+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T06:35:09.301717+0000","last_undegraded":"2026-02-18T06:35:09.301717+0000","last_fullsized":"2026-02-18T06:35:09.301717+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301392+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T06:35:09.301392+0000","last_peered":"2026-02-18T06:35:09.301392+0000","last_clean":"2026-02-18T06:35:09.301392+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T06:35:09.301392+0000","last_undegraded":"2026-02-18T06:35:09.301392+0000","last_fullsized":"2026-02-18T06:35:09.301392+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.300117+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T06:35:09.300117+0000","last_peered":"2026-02-18T06:35:09.300117+0000","last_clean":"2026-02-18T06:35:09.300117+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T06:35:09.300117+0000","last_undegraded":"2026-02-18T06:35:09.300117+0000","last_fullsized":"2026-02-18T06:35:09.300117+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.298112+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T06:35:09.298112+0000","last_peered":"2026-02-18T06:35:09.298112+0000","last_clean":"2026-02-18T06:35:09.298112+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T06:35:09.298112+0000","last_undegraded":"2026-02-18T06:35:09.298112+0000","last_fullsized":"2026-02-18T06:35:09.298112+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.298131+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T06:35:09.298131+0000","last_peered":"2026-02-18T06:35:09.298131+0000","last_clean":"2026-02-18T06:35:09.298131+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T06:35:09.298131+0000","last_undegraded":"2026-02-18T06:35:09.298131+0000","last_fullsized":"2026-02-18T06:35:09.298131+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301183+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T06:35:09.301183+0000","last_peered":"2026-02-18T06:35:09.301183+0000","last_clean":"2026-02-18T06:35:09.301183+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T06:35:09.301183+0000","last_undegraded":"2026-02-18T06:35:09.301183+0000","last_fullsized":"2026-02-18T06:35:09.301183+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":111,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.300749+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T06:35:09.300749+0000","last_peered":"2026-02-18T06:35:09.300749+0000","last_clean":"2026-02-18T06:35:09.300749+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T06:35:09.300749+0000","last_undegraded":"2026-02-18T06:35:09.300749+0000","last_fullsized":"2026-02-18T06:35:09.300749+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":131072,"data_stored":38199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8124,"internal_metadata":27582532},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":628586,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":628586,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T06:35:10.660 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T06:35:10.840 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:10.840 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T06:35:10.848 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-18T06:35:10.283088+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81384,"kb_used_data":504,"kb_used_omap":25,"kb_used_meta":80806,"kb_avail":283034136,"statfs":{"total":289910292480,"available":289826955264,"internally_reserved":0,"allocated":516096,"data_stored":1295371,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":25674,"internal_metadata":82746294},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.990622"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301780+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T06:35:09.301780+0000","last_peered":"2026-02-18T06:35:09.301780+0000","last_clean":"2026-02-18T06:35:09.301780+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T06:35:09.301780+0000","last_undegraded":"2026-02-18T06:35:09.301780+0000","last_fullsized":"2026-02-18T06:35:09.301780+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301102+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T06:35:09.301102+0000","last_peered":"2026-02-18T06:35:09.301102+0000","last_clean":"2026-02-18T06:35:09.301102+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T06:35:09.301102+0000","last_undegraded":"2026-02-18T06:35:09.301102+0000","last_fullsized":"2026-02-18T06:35:09.301102+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301717+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T06:35:09.301717+0000","last_peered":"2026-02-18T06:35:09.301717+0000","last_clean":"2026-02-18T06:35:09.301717+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T06:35:09.301717+0000","last_undegraded":"2026-02-18T06:35:09.301717+0000","last_fullsized":"2026-02-18T06:35:09.301717+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301392+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T06:35:09.301392+0000","last_peered":"2026-02-18T06:35:09.301392+0000","last_clean":"2026-02-18T06:35:09.301392+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T06:35:09.301392+0000","last_undegraded":"2026-02-18T06:35:09.301392+0000","last_fullsized":"2026-02-18T06:35:09.301392+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.300117+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T06:35:09.300117+0000","last_peered":"2026-02-18T06:35:09.300117+0000","last_clean":"2026-02-18T06:35:09.300117+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T06:35:09.300117+0000","last_undegraded":"2026-02-18T06:35:09.300117+0000","last_fullsized":"2026-02-18T06:35:09.300117+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.298112+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T06:35:09.298112+0000","last_peered":"2026-02-18T06:35:09.298112+0000","last_clean":"2026-02-18T06:35:09.298112+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T06:35:09.298112+0000","last_undegraded":"2026-02-18T06:35:09.298112+0000","last_fullsized":"2026-02-18T06:35:09.298112+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.298131+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T06:35:09.298131+0000","last_peered":"2026-02-18T06:35:09.298131+0000","last_clean":"2026-02-18T06:35:09.298131+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T06:35:09.298131+0000","last_undegraded":"2026-02-18T06:35:09.298131+0000","last_fullsized":"2026-02-18T06:35:09.298131+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.301183+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T06:35:09.301183+0000","last_peered":"2026-02-18T06:35:09.301183+0000","last_clean":"2026-02-18T06:35:09.301183+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T06:35:09.301183+0000","last_undegraded":"2026-02-18T06:35:09.301183+0000","last_fullsized":"2026-02-18T06:35:09.301183+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":111,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-18T06:35:09.300749+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T06:35:09.300749+0000","last_peered":"2026-02-18T06:35:09.300749+0000","last_clean":"2026-02-18T06:35:09.300749+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T06:35:09.300749+0000","last_undegraded":"2026-02-18T06:35:09.300749+0000","last_fullsized":"2026-02-18T06:35:09.300749+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":131072,"data_stored":38199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8124,"internal_metadata":27582532},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":628586,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":628586,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T06:35:10.850 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-18T06:35:10.850 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-18T06:35:10.850 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-18T06:35:10.850 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-18T06:35:11.052 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:11.053 INFO:teuthology.orchestra.run.trial006.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-18T06:35:11.061 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-18T06:35:11.062 INFO:teuthology.run_tasks:Running task ceph... 2026-02-18T06:35:11.077 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-18T06:35:11.077 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-18T06:35:11.077 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909', 'branch': '20.2.1_2.17.26-baseline_2', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '54909', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': '20.2.1_2.17.26-baseline_2'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': '20.2.1_2.17.26-baseline_2', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7878, 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': '20.2.1_2.17.26-baseline_2', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_adc1923395f2bcd6ae89b5d331d278364d1848ed/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed', 'targets': {'trial006.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPP05jXG1JMiRsC5EKWKC4JiRMBDH/oEdWq9Zi4ru9TMCkTxM7/9hlIOEHzP+L3FV2XDq89oJpv4o0bMYCJe/V8=', 'trial051.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLXWP8acdBulI9g3phySvn7Yy8qF7TW7SBOxFGID5vjvI+2GKGhwlNx0Ib+pGTj2SVsuhiqbtgtA24CXqUWxQak='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'adc1923395f2bcd6ae89b5d331d278364d1848ed'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-17_21:26:40', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3682'} 2026-02-18T06:35:11.078 DEBUG:teuthology.orchestra.run.trial006:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-18T06:35:11.080 DEBUG:teuthology.orchestra.run.trial051:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-18T06:35:11.098 DEBUG:teuthology.orchestra.run.trial006:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T06:35:11.136 DEBUG:teuthology.orchestra.run.trial051:> sudo install -d -m0777 -- /var/run/ceph 2026-02-18T06:35:11.162 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:11.162 DEBUG:teuthology.orchestra.run.trial051:> dd if=/scratch_devs of=/dev/stdout 2026-02-18T06:35:11.218 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-18T06:35:11.218 DEBUG:teuthology.orchestra.run.trial051:> stat /dev/vg_nvme/lv_1 2026-02-18T06:35:11.274 INFO:teuthology.orchestra.run.trial051.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Device: 6h/6d Inode: 1294 Links: 1 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Access: 2026-02-18 06:34:40.251599283 +0000 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Modify: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout:Change: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.275 INFO:teuthology.orchestra.run.trial051.stdout: Birth: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.276 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-18T06:35:11.338 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records in 2026-02-18T06:35:11.338 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records out 2026-02-18T06:35:11.338 INFO:teuthology.orchestra.run.trial051.stderr:512 bytes copied, 0.000146832 s, 3.5 MB/s 2026-02-18T06:35:11.340 DEBUG:teuthology.orchestra.run.trial051:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-18T06:35:11.398 DEBUG:teuthology.orchestra.run.trial051:> stat /dev/vg_nvme/lv_2 2026-02-18T06:35:11.454 INFO:teuthology.orchestra.run.trial051.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Device: 6h/6d Inode: 1290 Links: 1 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Access: 2026-02-18 06:34:40.252599258 +0000 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Modify: 2026-02-18 06:33:52.983758467 +0000 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout:Change: 2026-02-18 06:33:52.983758467 +0000 2026-02-18T06:35:11.455 INFO:teuthology.orchestra.run.trial051.stdout: Birth: 2026-02-18 06:33:52.983758467 +0000 2026-02-18T06:35:11.456 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-18T06:35:11.518 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records in 2026-02-18T06:35:11.519 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records out 2026-02-18T06:35:11.519 INFO:teuthology.orchestra.run.trial051.stderr:512 bytes copied, 0.000136552 s, 3.7 MB/s 2026-02-18T06:35:11.520 DEBUG:teuthology.orchestra.run.trial051:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-18T06:35:11.576 DEBUG:teuthology.orchestra.run.trial051:> stat /dev/vg_nvme/lv_3 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Device: 6h/6d Inode: 1311 Links: 1 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Access: 2026-02-18 06:34:40.252599258 +0000 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Modify: 2026-02-18 06:33:52.987758368 +0000 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout:Change: 2026-02-18 06:33:52.987758368 +0000 2026-02-18T06:35:11.634 INFO:teuthology.orchestra.run.trial051.stdout: Birth: 2026-02-18 06:33:52.987758368 +0000 2026-02-18T06:35:11.634 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-18T06:35:11.696 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records in 2026-02-18T06:35:11.696 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records out 2026-02-18T06:35:11.696 INFO:teuthology.orchestra.run.trial051.stderr:512 bytes copied, 0.000151941 s, 3.4 MB/s 2026-02-18T06:35:11.697 DEBUG:teuthology.orchestra.run.trial051:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-18T06:35:11.755 DEBUG:teuthology.orchestra.run.trial051:> stat /dev/vg_nvme/lv_4 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Device: 6h/6d Inode: 1304 Links: 1 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Context: system_u:object_r:device_t:s0 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Access: 2026-02-18 06:34:40.252599258 +0000 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Modify: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout:Change: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.813 INFO:teuthology.orchestra.run.trial051.stdout: Birth: 2026-02-18 06:33:52.984758442 +0000 2026-02-18T06:35:11.813 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-18T06:35:11.876 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records in 2026-02-18T06:35:11.876 INFO:teuthology.orchestra.run.trial051.stderr:1+0 records out 2026-02-18T06:35:11.876 INFO:teuthology.orchestra.run.trial051.stderr:512 bytes copied, 0.000145809 s, 3.5 MB/s 2026-02-18T06:35:11.877 DEBUG:teuthology.orchestra.run.trial051:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-18T06:35:11.933 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:35:11.933 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial051.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T06:35:11.933 INFO:tasks.ceph:Generating config... 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client] log to stderr = False 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:11.934 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:11.935 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:11.935 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-18T06:35:11.935 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-18T06:35:11.935 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-18T06:35:11.935 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-18T06:35:11.935 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-18T06:35:11.935 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] bluestore compression algorithm = zlib 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-18T06:35:11.935 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-18T06:35:11.936 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-18T06:35:11.936 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-18T06:35:11.936 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-18T06:35:11.936 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-18T06:35:11.936 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.006 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.008 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.086 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.152 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.51')] 2026-02-18T06:35:12.153 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.51', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'zlib', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-18T06:35:12.154 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:12.155 DEBUG:teuthology.orchestra.run.trial051:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T06:35:12.211 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.51 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:12.281 INFO:teuthology.orchestra.run.trial051.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:12.281 INFO:teuthology.orchestra.run.trial051.stdout:monmaptool: generated fsid 9115e2ce-4d18-4172-bcd4-e2000f709e06 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:setting min_mon_release = tentacle 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:epoch 0 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:fsid 9115e2ce-4d18-4172-bcd4-e2000f709e06 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:last_changed 2026-02-18T06:35:12.279974+0000 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:created 2026-02-18T06:35:12.279974+0000 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:min_mon_release 20 (tentacle) 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:election_strategy: 1 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:0: [v2:10.20.193.51:3300/0,v1:10.20.193.51:6789/0] mon.a 2026-02-18T06:35:12.282 INFO:teuthology.orchestra.run.trial051.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-18T06:35:12.284 DEBUG:teuthology.orchestra.run.trial051:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-18T06:35:12.340 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 9115e2ce-4d18-4172-bcd4-e2000f709e06... 2026-02-18T06:35:12.341 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-18T06:35:12.343 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-18T06:35:12.383 INFO:teuthology.orchestra.run.trial006.stdout:[global] 2026-02-18T06:35:12.383 INFO:teuthology.orchestra.run.trial006.stdout: chdir = "" 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: auth supported = cephx 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: filestore xattr use omap = true 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: mon clock drift allowed = 1.000 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: osd crush chooseleaf type = 0 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: auth debug = true 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.384 INFO:teuthology.orchestra.run.trial006.stdout: ms die on old message = true 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: ms die on bug = true 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: mon pg warn max object skew = 0 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: osd pool default size = 2 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: mon osd allow primary affinity = true 2026-02-18T06:35:12.385 INFO:teuthology.orchestra.run.trial006.stdout: mon osd allow pg remap = true 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on legacy crush tunables = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on crush straw calc version zero = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on no sortbitwise = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on osd down out interval zero = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on too few osds = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: mon_allow_pool_size_one = true 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.386 INFO:teuthology.orchestra.run.trial006.stdout: osd default data pool replay window = 5 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon allow pool delete = true 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon cluster log file level = debug 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: debug asserts on shutdown = true 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon health detail to clog = false 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon host = 10.20.193.51 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon client directed command retry = 5 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: mon warn on pool no app = False 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: ms inject socket failures = 5000 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: fsid = 9115e2ce-4d18-4172-bcd4-e2000f709e06 2026-02-18T06:35:12.387 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout:[osd] 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd journal size = 100 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd scrub load threshold = 5.0 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd scrub max interval = 600 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd mclock profile = high_recovery_ops 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd mclock skip benchmark = true 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd recover clone overlap = true 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd recovery max chunk = 1048576 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.388 INFO:teuthology.orchestra.run.trial006.stdout: osd debug shutdown = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd debug op order = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd debug verify stray on activate = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd debug trim objects = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd open classes on start = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd debug pg log writeout = true 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd deep scrub update digest min age = 30 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: osd map max advance = 10 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.389 INFO:teuthology.orchestra.run.trial006.stdout: journal zero on create = true 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: filestore ondisk finisher threads = 3 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: filestore apply finisher threads = 3 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: bdev debug aio = true 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: osd debug misdirected ops = true 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: bluestore block size = 96636764160 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: bluestore compression algorithm = zlib 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: bluestore compression mode = aggressive 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: bluestore fsck on mount = True 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: debug bluefs = 1/20 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: debug bluestore = 1/20 2026-02-18T06:35:12.390 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug osd = 20 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug rocksdb = 4/10 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: mon osd full ratio = 0.9 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: mon osd nearfull ratio = 0.8 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: osd failsafe full ratio = 0.95 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: osd objectstore = bluestore 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout:[mgr] 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug mgr = 20 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug mon = 20 2026-02-18T06:35:12.391 INFO:teuthology.orchestra.run.trial006.stdout: debug auth = 20 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mgr/telemetry/nag = false 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout:[mon] 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: debug ms = 1 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: debug mon = 20 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: debug paxos = 20 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: debug auth = 20 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mon data avail warn = 5 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mon mgr mkfs grace = 240 2026-02-18T06:35:12.392 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: mon osd reporter subtree level = osd 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: mon osd prime pg temp = true 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T06:35:12.393 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: # 1m isn't quite enough 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: mon_down_mkfs_grace = 2m 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: mon_warn_on_filestore_osds = false 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout:[client] 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: rgw cache enabled = true 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: rgw enable ops log = true 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: rgw enable usage log = true 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T06:35:12.394 INFO:teuthology.orchestra.run.trial006.stdout: debug rbd = 20 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: debug rbd_mirror = 15 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: log to stderr = False 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: rbd default features = 125 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: rbd mirror image policy type = simple 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout:[mon.a] 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.0] 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.1] 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.2] 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.395 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.396 INFO:teuthology.orchestra.run.trial006.stdout:[client.mirror.3] 2026-02-18T06:35:12.396 INFO:teuthology.orchestra.run.trial006.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.396 INFO:teuthology.orchestra.run.trial006.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout:[global] 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: chdir = "" 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: auth supported = cephx 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: filestore xattr use omap = true 2026-02-18T06:35:12.418 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: mon clock drift allowed = 1.000 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: osd crush chooseleaf type = 0 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: auth debug = true 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: ms die on old message = true 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: ms die on bug = true 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: mon pg warn max object skew = 0 2026-02-18T06:35:12.419 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: # disable pg_autoscaler by default for new pools 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: osd pool default size = 2 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon osd allow primary affinity = true 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon osd allow pg remap = true 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on legacy crush tunables = false 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on crush straw calc version zero = false 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on no sortbitwise = false 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on osd down out interval zero = false 2026-02-18T06:35:12.420 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on too few osds = false 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: mon_allow_pool_size_one = true 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: osd default data pool replay window = 5 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: mon allow pool delete = true 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: mon cluster log file level = debug 2026-02-18T06:35:12.421 INFO:teuthology.orchestra.run.trial051.stdout: debug asserts on shutdown = true 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: mon health detail to clog = false 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: mon host = 10.20.193.51 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: mon client directed command retry = 5 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: mon warn on pool no app = False 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: ms inject socket failures = 5000 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: fsid = 9115e2ce-4d18-4172-bcd4-e2000f709e06 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout:[osd] 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: osd journal size = 100 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: osd scrub load threshold = 5.0 2026-02-18T06:35:12.422 INFO:teuthology.orchestra.run.trial051.stdout: osd scrub max interval = 600 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd mclock profile = high_recovery_ops 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd mclock skip benchmark = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd recover clone overlap = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd recovery max chunk = 1048576 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd debug shutdown = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd debug op order = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd debug verify stray on activate = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd debug trim objects = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd open classes on start = true 2026-02-18T06:35:12.423 INFO:teuthology.orchestra.run.trial051.stdout: osd debug pg log writeout = true 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: osd deep scrub update digest min age = 30 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: osd map max advance = 10 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: journal zero on create = true 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: filestore ondisk finisher threads = 3 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: filestore apply finisher threads = 3 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.424 INFO:teuthology.orchestra.run.trial051.stdout: bdev debug aio = true 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: osd debug misdirected ops = true 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: bluestore block size = 96636764160 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: bluestore compression algorithm = zlib 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: bluestore compression mode = aggressive 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: bluestore fsck on mount = True 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: debug bluefs = 1/20 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: debug bluestore = 1/20 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: debug osd = 20 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: debug rocksdb = 4/10 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: mon osd backfillfull_ratio = 0.85 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: mon osd full ratio = 0.9 2026-02-18T06:35:12.425 INFO:teuthology.orchestra.run.trial051.stdout: mon osd nearfull ratio = 0.8 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: osd failsafe full ratio = 0.95 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: osd objectstore = bluestore 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout:[mgr] 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: debug mgr = 20 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: debug mon = 20 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: debug auth = 20 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: mgr/telemetry/nag = false 2026-02-18T06:35:12.426 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout:[mon] 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: debug ms = 1 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: debug mon = 20 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: debug paxos = 20 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: debug auth = 20 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon data avail warn = 5 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon mgr mkfs grace = 240 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min pgs per osd = 4 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon osd reporter subtree level = osd 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon osd prime pg temp = true 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: mon reweight min bytes per osd = 10 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.427 INFO:teuthology.orchestra.run.trial051.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: auth mon ticket ttl = 660 # 11m 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: auth service ticket ttl = 240 # 4m 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: # don't complain about insecure global_id in the test suite 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: # 1m isn't quite enough 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: mon_down_mkfs_grace = 2m 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: mon_warn_on_filestore_osds = false 2026-02-18T06:35:12.428 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout:[client] 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: rgw cache enabled = true 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: rgw enable ops log = true 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: rgw enable usage log = true 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: debug rbd = 20 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: debug rbd_mirror = 15 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: log to stderr = False 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: rbd default features = 125 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout: rbd mirror image policy type = simple 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout:[mon.a] 2026-02-18T06:35:12.429 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.0] 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.1] 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.2] 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.3] 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-18T06:35:12.430 INFO:teuthology.orchestra.run.trial051.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-18T06:35:12.431 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-18T06:35:12.432 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.513 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-18T06:35:12.514 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:12.514 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-18T06:35:12.572 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:12.572 DEBUG:teuthology.orchestra.run.trial051:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-18T06:35:12.627 INFO:tasks.ceph:Sending monmap to node ubuntu@trial006.front.sepia.ceph.com 2026-02-18T06:35:12.628 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:35:12.628 DEBUG:teuthology.orchestra.run.trial006:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-18T06:35:12.628 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.662 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:35:12.662 DEBUG:teuthology.orchestra.run.trial006:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:12.718 INFO:tasks.ceph:Sending monmap to node ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:35:12.719 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:12.719 DEBUG:teuthology.orchestra.run.trial051:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-18T06:35:12.719 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-18T06:35:12.752 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:12.753 DEBUG:teuthology.orchestra.run.trial051:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:12.809 INFO:tasks.ceph:Setting up mon nodes... 2026-02-18T06:35:12.809 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-18T06:35:12.809 DEBUG:teuthology.orchestra.run.trial006:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-18T06:35:12.854 INFO:teuthology.orchestra.run.trial006.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-18T06:35:12.857 INFO:tasks.ceph:Setting up mds nodes... 2026-02-18T06:35:12.857 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-18T06:35:12.857 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-18T06:35:12.891 INFO:teuthology.orchestra.run.trial006.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-18T06:35:12.902 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-18T06:35:12.938 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-18T06:35:12.949 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-18T06:35:13.022 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-18T06:35:13.033 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-18T06:35:13.108 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-18T06:35:13.119 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-18T06:35:13.192 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-18T06:35:13.203 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-18T06:35:13.278 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-18T06:35:13.288 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-18T06:35:13.362 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-18T06:35:13.372 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-18T06:35:13.445 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-18T06:35:13.456 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-18T06:35:13.529 INFO:teuthology.orchestra.run.trial051.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-18T06:35:13.540 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-18T06:35:13.540 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial006.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial051.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-18T06:35:13.541 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-18T06:35:13.605 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:35:13.605 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-18T06:35:13.605 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:35:13.605 DEBUG:teuthology.orchestra.run.trial051:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-18T06:35:13.670 INFO:teuthology.orchestra.run.trial051.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:35:13.670 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:35:13.670 INFO:teuthology.orchestra.run.trial051.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:35:13.670 INFO:teuthology.orchestra.run.trial051.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:35:13.671 INFO:teuthology.orchestra.run.trial051.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:35:14.240 INFO:teuthology.orchestra.run.trial051.stdout:Discarding blocks...Done. 2026-02-18T06:35:14.242 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial051.front.sepia.ceph.com -o noatime 2026-02-18T06:35:14.242 DEBUG:teuthology.orchestra.run.trial051:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-18T06:35:14.314 DEBUG:teuthology.orchestra.run.trial051:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-18T06:35:14.381 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-18T06:35:14.446 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:35:14.447 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-18T06:35:14.447 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:35:14.447 DEBUG:teuthology.orchestra.run.trial051:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-18T06:35:14.511 INFO:teuthology.orchestra.run.trial051.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:35:14.512 INFO:teuthology.orchestra.run.trial051.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:35:15.073 INFO:teuthology.orchestra.run.trial051.stdout:Discarding blocks...Done. 2026-02-18T06:35:15.075 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial051.front.sepia.ceph.com -o noatime 2026-02-18T06:35:15.075 DEBUG:teuthology.orchestra.run.trial051:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-18T06:35:15.142 DEBUG:teuthology.orchestra.run.trial051:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-18T06:35:15.209 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-18T06:35:15.274 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-18T06:35:15.275 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-18T06:35:15.275 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:35:15.275 DEBUG:teuthology.orchestra.run.trial051:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout: = sunit=0 swidth=0 blks 2026-02-18T06:35:15.341 INFO:teuthology.orchestra.run.trial051.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-18T06:35:15.342 INFO:teuthology.orchestra.run.trial051.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-18T06:35:15.342 INFO:teuthology.orchestra.run.trial051.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-18T06:35:15.342 INFO:teuthology.orchestra.run.trial051.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-18T06:35:15.898 INFO:teuthology.orchestra.run.trial051.stdout:Discarding blocks...Done. 2026-02-18T06:35:15.900 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial051.front.sepia.ceph.com -o noatime 2026-02-18T06:35:15.900 DEBUG:teuthology.orchestra.run.trial051:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-18T06:35:15.967 DEBUG:teuthology.orchestra.run.trial051:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-18T06:35:16.034 DEBUG:teuthology.orchestra.run.trial051:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:16.111 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.108+0000 7fadf91c6900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-18T06:35:16.111 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.108+0000 7fadf91c6900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-18T06:35:16.111 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.108+0000 7fadf91c6900 -1 bdev(0x5590ecfc9800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-18T06:35:16.112 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.108+0000 7fadf91c6900 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-18T06:35:16.549 DEBUG:teuthology.orchestra.run.trial051:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-18T06:35:16.614 DEBUG:teuthology.orchestra.run.trial051:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:16.691 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.688+0000 7fc78a414900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-18T06:35:16.691 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.688+0000 7fc78a414900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-18T06:35:16.692 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.688+0000 7fc78a414900 -1 bdev(0x55c302d89800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-18T06:35:16.692 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:16.688+0000 7fc78a414900 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-18T06:35:17.133 DEBUG:teuthology.orchestra.run.trial051:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-18T06:35:17.199 DEBUG:teuthology.orchestra.run.trial051:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:17.276 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:17.273+0000 7f21516e0900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-18T06:35:17.276 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:17.273+0000 7f21516e0900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-18T06:35:17.277 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:17.273+0000 7f21516e0900 -1 bdev(0x564398f8b800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-18T06:35:17.277 INFO:teuthology.orchestra.run.trial051.stderr:2026-02-18T06:35:17.273+0000 7f21516e0900 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-18T06:35:17.730 DEBUG:teuthology.orchestra.run.trial051:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-18T06:35:17.796 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-18T06:35:17.796 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:35:17.796 DEBUG:teuthology.orchestra.run.trial006:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-18T06:35:17.822 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:17.823 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-18T06:35:17.859 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:17.860 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-18T06:35:17.924 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:17.924 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-18T06:35:17.988 DEBUG:teuthology.orchestra.run.trial006:> set -ex 2026-02-18T06:35:17.989 DEBUG:teuthology.orchestra.run.trial006:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-18T06:35:18.007 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.007 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-18T06:35:18.045 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.045 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-18T06:35:18.101 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.101 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-18T06:35:18.157 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.157 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-18T06:35:18.213 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.213 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-18T06:35:18.269 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.269 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-18T06:35:18.325 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.325 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-18T06:35:18.381 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:18.381 DEBUG:teuthology.orchestra.run.trial051:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-18T06:35:18.437 INFO:tasks.ceph:Adding keys to all mons... 2026-02-18T06:35:18.437 DEBUG:teuthology.orchestra.run.trial051:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-18T06:35:18.501 INFO:teuthology.orchestra.run.trial051.stdout:[mgr.x] 2026-02-18T06:35:18.501 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCgXZVpmQTnMhAAnNa7Rc1aLwSCo+oTxApBeQ== 2026-02-18T06:35:18.501 INFO:teuthology.orchestra.run.trial051.stdout:[osd.0] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCkXZVpQ9yIBhAAz5JmdTi5VRnNsIKys1ZHDA== 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout:[osd.1] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCkXZVpiAweKRAAqr2oCaNi8GKUKqd5KPcuOQ== 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout:[osd.2] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQClXZVpP0ViEBAAB/G5UnM1c3PD8Rh4u0FCzg== 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout:[client.0] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCgXZVpWwMdNRAAaY3znJYQEAq8ff3MxVcuGw== 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCgXZVpQxTVNxAAbDDSb26swyUDnjrF0TrH1A== 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.0] 2026-02-18T06:35:18.502 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpkl5CARAANlcXg7ztyKjiNRyV0kpqHg== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.1] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVp/r9cBhAA77Ii1SOaRxahWR7NyRGfrA== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.2] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVp0phoCxAAyM4Ct43JguV9lgb0RXIuVQ== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.3] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpnNmGEBAAe2O1FtE8VHCbVc9SiMSH0w== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.4] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpH3OBFRAAZLtlkibzczN56OoW8dOmhA== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.5] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVputh+GhAARrjQkeZ+nmf6yOgZWn0RXw== 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.6] 2026-02-18T06:35:18.503 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpEhF7HxAAYwnoxXWjGOiL50mskrX/Zg== 2026-02-18T06:35:18.505 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-18T06:35:18.585 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:35:18.665 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:35:18.745 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-18T06:35:18.824 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:18.905 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:18.987 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.066 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.146 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.226 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.308 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.389 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.469 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-18T06:35:19.549 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-18T06:35:19.549 DEBUG:teuthology.orchestra.run.trial051:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-18T06:35:19.612 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-18T06:35:19.695 DEBUG:teuthology.orchestra.run.trial051:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-18T06:35:19.760 DEBUG:teuthology.orchestra.run.trial051:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-18T06:35:19.816 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-18T06:35:19.816 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-18T06:35:19.817 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-18T06:35:19.858 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-18T06:35:19.858 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-18T06:35:19.859 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-18T06:35:19.859 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-18T06:35:19.862 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-18T06:35:19.862 DEBUG:tasks.ceph:set 0 configs 2026-02-18T06:35:19.862 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 config dump 2026-02-18T06:35:19.960 INFO:teuthology.orchestra.run.trial051.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-18T06:35:19.968 INFO:tasks.ceph:Setting crush tunables to default 2026-02-18T06:35:19.969 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-18T06:35:20.072 INFO:teuthology.orchestra.run.trial051.stderr:adjusted tunables profile to default 2026-02-18T06:35:20.081 INFO:tasks.ceph:check_enable_crimson: False 2026-02-18T06:35:20.081 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-18T06:35:20.081 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:20.081 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-18T06:35:20.106 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:20.106 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-18T06:35:20.170 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:20.170 DEBUG:teuthology.orchestra.run.trial051:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-18T06:35:20.233 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 osd new e9c2b530-d1be-41b5-9131-32932905caee 0 2026-02-18T06:35:20.377 INFO:teuthology.orchestra.run.trial051.stdout:0 2026-02-18T06:35:20.385 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 osd new db1cbb05-e951-4302-b391-ec61dd00ce9b 1 2026-02-18T06:35:20.487 INFO:teuthology.orchestra.run.trial051.stdout:1 2026-02-18T06:35:20.494 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 osd new fe301036-a947-4204-87a7-3efc38575b5d 2 2026-02-18T06:35:20.598 INFO:teuthology.orchestra.run.trial051.stdout:2 2026-02-18T06:35:20.608 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-18T06:35:20.608 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-18T06:35:20.610 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-18T06:35:20.610 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-18T06:35:20.610 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-18T06:35:20.613 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-18T06:35:20.613 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-18T06:35:20.613 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-18T06:35:20.615 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-18T06:35:20.615 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T06:35:20.617 INFO:tasks.ceph.cluster2.mgr.x.trial006.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-18T06:35:20.617 INFO:tasks.ceph.cluster2.mgr.x.trial006.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-18T06:35:20.617 INFO:tasks.ceph.cluster2.mgr.x.trial006.stderr: from numpy import show_config as show_numpy_config 2026-02-18T06:35:20.715 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:20.716 INFO:teuthology.orchestra.run.trial051.stdout:{"epoch":5,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T06:35:20.597066+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:20.723 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-18T06:35:20.723 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-18T06:35:20.848 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:35:20.845+0000 7f8d9b093900 -1 Falling back to public interface 2026-02-18T06:35:20.849 INFO:tasks.ceph.cluster2.osd.1.trial051.stderr:2026-02-18T06:35:20.846+0000 7fdaf00b2900 -1 Falling back to public interface 2026-02-18T06:35:20.863 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:35:20.860+0000 7ff40560f900 -1 Falling back to public interface 2026-02-18T06:35:21.026 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-18T06:35:21.117 INFO:teuthology.misc.health.trial051.stdout: 2026-02-18T06:35:21.118 INFO:teuthology.misc.health.trial051.stdout:{"epoch":5,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T06:35:20.597066+0000","last_up_change":"0.000000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:21.123 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-18T06:35:21.385 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:35:21.382+0000 7f8d9b093900 -1 osd.2 0 log_to_monitors true 2026-02-18T06:35:21.399 INFO:tasks.ceph.cluster2.osd.1.trial051.stderr:2026-02-18T06:35:21.396+0000 7fdaf00b2900 -1 osd.1 0 log_to_monitors true 2026-02-18T06:35:21.442 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:35:21.439+0000 7ff40560f900 -1 osd.0 0 log_to_monitors true 2026-02-18T06:35:22.809 INFO:tasks.ceph.cluster2.mgr.x.trial006.stderr:2026-02-18T06:35:22.807+0000 7fa6fb295640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-18T06:35:22.940 INFO:tasks.ceph.cluster2.osd.1.trial051.stderr:2026-02-18T06:35:22.937+0000 7fdaec040640 -1 osd.1 0 waiting for initial osdmap 2026-02-18T06:35:22.940 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:35:22.937+0000 7f8d97021640 -1 osd.2 0 waiting for initial osdmap 2026-02-18T06:35:22.940 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:35:22.937+0000 7ff400d76640 -1 osd.0 0 waiting for initial osdmap 2026-02-18T06:35:22.941 INFO:tasks.ceph.cluster2.osd.1.trial051.stderr:2026-02-18T06:35:22.939+0000 7fdae363e640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:22.942 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:35:22.939+0000 7ff3f8374640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:22.942 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:35:22.939+0000 7f8d8e61f640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-18T06:35:27.426 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-18T06:35:27.607 INFO:teuthology.misc.health.trial051.stdout: 2026-02-18T06:35:27.608 INFO:teuthology.misc.health.trial051.stdout:{"epoch":10,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T06:35:26.930779+0000","last_up_change":"2026-02-18T06:35:23.923562+0000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:25.821503+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6816","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6817","nonce":3495423820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6818","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6819","nonce":3495423820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6822","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6823","nonce":3495423820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6820","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6821","nonce":3495423820}]},"public_addr":"10.20.193.51:6817/3495423820","cluster_addr":"10.20.193.51:6819/3495423820","heartbeat_back_addr":"10.20.193.51:6823/3495423820","heartbeat_front_addr":"10.20.193.51:6821/3495423820","state":["exists","up"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6808","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6809","nonce":1672938220}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6810","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6811","nonce":1672938220}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6814","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6815","nonce":1672938220}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6812","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6813","nonce":1672938220}]},"public_addr":"10.20.193.51:6809/1672938220","cluster_addr":"10.20.193.51:6811/1672938220","heartbeat_back_addr":"10.20.193.51:6815/1672938220","heartbeat_front_addr":"10.20.193.51:6813/1672938220","state":["exists","up"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6800","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6801","nonce":3566361000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6802","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6803","nonce":3566361000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6806","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6807","nonce":3566361000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6804","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6805","nonce":3566361000}]},"public_addr":"10.20.193.51:6801/3566361000","cluster_addr":"10.20.193.51:6803/3566361000","heartbeat_back_addr":"10.20.193.51:6807/3566361000","heartbeat_front_addr":"10.20.193.51:6805/3566361000","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:27.616 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-18T06:35:27.616 INFO:tasks.ceph:Creating RBD pool 2026-02-18T06:35:27.617 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-18T06:35:27.936 INFO:teuthology.orchestra.run.trial051.stderr:pool 'rbd' created 2026-02-18T06:35:27.946 DEBUG:teuthology.orchestra.run.trial051:> rbd --cluster cluster2 pool init rbd 2026-02-18T06:35:30.948 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-18T06:35:30.949 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-18T06:35:30.950 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-18T06:35:31.134 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:31.144 INFO:teuthology.orchestra.run.trial051.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-18T06:35:31.145 INFO:tasks.ceph_manager:config epoch is 1 2026-02-18T06:35:31.145 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-18T06:35:31.145 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-18T06:35:31.145 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-18T06:35:31.358 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:31.366 INFO:teuthology.orchestra.run.trial051.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6826","nonce":443170159},{"type":"v1","addr":"10.20.193.6:6827","nonce":443170159}]},"active_addr":"10.20.193.6:6827/443170159","active_change":"2026-02-18T06:35:21.801210+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":1195146218}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":2907911515}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":1985841212}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.6:0","nonce":2790507274}]}]} 2026-02-18T06:35:31.370 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-18T06:35:31.370 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-18T06:35:31.370 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T06:35:31.551 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:31.551 INFO:teuthology.orchestra.run.trial051.stdout:{"epoch":14,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T06:35:30.937859+0000","last_up_change":"2026-02-18T06:35:23.923562+0000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:25.821503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:27.797884+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6816","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6817","nonce":3495423820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6818","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6819","nonce":3495423820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6822","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6823","nonce":3495423820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6820","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6821","nonce":3495423820}]},"public_addr":"10.20.193.51:6817/3495423820","cluster_addr":"10.20.193.51:6819/3495423820","heartbeat_back_addr":"10.20.193.51:6823/3495423820","heartbeat_front_addr":"10.20.193.51:6821/3495423820","state":["exists","up"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6808","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6809","nonce":1672938220}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6810","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6811","nonce":1672938220}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6814","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6815","nonce":1672938220}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6812","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6813","nonce":1672938220}]},"public_addr":"10.20.193.51:6809/1672938220","cluster_addr":"10.20.193.51:6811/1672938220","heartbeat_back_addr":"10.20.193.51:6815/1672938220","heartbeat_front_addr":"10.20.193.51:6813/1672938220","state":["exists","up"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6800","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6801","nonce":3566361000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6802","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6803","nonce":3566361000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6806","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6807","nonce":3566361000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6804","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6805","nonce":3566361000}]},"public_addr":"10.20.193.51:6801/3566361000","cluster_addr":"10.20.193.51:6803/3566361000","heartbeat_back_addr":"10.20.193.51:6807/3566361000","heartbeat_front_addr":"10.20.193.51:6805/3566361000","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:31.561 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-18T06:35:31.561 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T06:35:31.742 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:31.742 INFO:teuthology.orchestra.run.trial051.stdout:{"epoch":14,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T06:35:30.937859+0000","last_up_change":"2026-02-18T06:35:23.923562+0000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:25.821503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:27.797884+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6816","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6817","nonce":3495423820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6818","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6819","nonce":3495423820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6822","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6823","nonce":3495423820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6820","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6821","nonce":3495423820}]},"public_addr":"10.20.193.51:6817/3495423820","cluster_addr":"10.20.193.51:6819/3495423820","heartbeat_back_addr":"10.20.193.51:6823/3495423820","heartbeat_front_addr":"10.20.193.51:6821/3495423820","state":["exists","up"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6808","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6809","nonce":1672938220}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6810","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6811","nonce":1672938220}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6814","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6815","nonce":1672938220}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6812","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6813","nonce":1672938220}]},"public_addr":"10.20.193.51:6809/1672938220","cluster_addr":"10.20.193.51:6811/1672938220","heartbeat_back_addr":"10.20.193.51:6815/1672938220","heartbeat_front_addr":"10.20.193.51:6813/1672938220","state":["exists","up"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6800","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6801","nonce":3566361000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6802","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6803","nonce":3566361000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6806","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6807","nonce":3566361000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6804","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6805","nonce":3566361000}]},"public_addr":"10.20.193.51:6801/3566361000","cluster_addr":"10.20.193.51:6803/3566361000","heartbeat_back_addr":"10.20.193.51:6807/3566361000","heartbeat_front_addr":"10.20.193.51:6805/3566361000","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T06:35:31.752 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-18T06:35:31.752 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-18T06:35:31.753 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-18T06:35:31.842 INFO:teuthology.orchestra.run.trial051.stdout:34359738372 2026-02-18T06:35:31.843 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T06:35:31.844 INFO:teuthology.orchestra.run.trial051.stdout:34359738372 2026-02-18T06:35:31.845 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T06:35:31.846 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:31.846 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T06:35:32.068 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:32.069 INFO:teuthology.orchestra.run.trial051.stdout:34359738370 2026-02-18T06:35:32.071 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:32.079 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-02-18T06:35:32.080 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-02-18T06:35:32.082 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-18T06:35:33.080 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T06:35:33.081 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T06:35:33.082 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T06:35:33.265 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:33.269 INFO:teuthology.orchestra.run.trial051.stdout:34359738370 2026-02-18T06:35:33.275 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-02-18T06:35:33.281 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-02-18T06:35:33.301 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:33.312 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-18T06:35:34.276 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-18T06:35:34.281 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-18T06:35:34.313 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-18T06:35:34.463 INFO:teuthology.orchestra.run.trial051.stdout:34359738372 2026-02-18T06:35:34.474 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.1 2026-02-18T06:35:34.475 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:34.502 INFO:teuthology.orchestra.run.trial051.stdout:34359738372 2026-02-18T06:35:34.509 INFO:teuthology.orchestra.run.trial051.stdout:34359738371 2026-02-18T06:35:34.513 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-18T06:35:34.513 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:34.520 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.2 2026-02-18T06:35:34.520 DEBUG:teuthology.parallel:result is None 2026-02-18T06:35:34.521 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-18T06:35:34.521 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T06:35:34.737 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:34.738 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T06:35:34.747 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-02-18T06:35:33.807188+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81368,"kb_used_data":472,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283034152,"statfs":{"total":289910292480,"available":289826971648,"internally_reserved":0,"allocated":483328,"data_stored":1283458,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":23723,"internal_metadata":82748245},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.868594"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940363+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T06:35:30.940363+0000","last_peered":"2026-02-18T06:35:30.940363+0000","last_clean":"2026-02-18T06:35:30.940363+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T06:35:30.940363+0000","last_undegraded":"2026-02-18T06:35:30.940363+0000","last_fullsized":"2026-02-18T06:35:30.940363+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939834+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T06:35:30.939834+0000","last_peered":"2026-02-18T06:35:30.939834+0000","last_clean":"2026-02-18T06:35:30.939834+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T06:35:30.939834+0000","last_undegraded":"2026-02-18T06:35:30.939834+0000","last_fullsized":"2026-02-18T06:35:30.939834+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940102+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T06:35:30.940102+0000","last_peered":"2026-02-18T06:35:30.940102+0000","last_clean":"2026-02-18T06:35:30.940102+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T06:35:30.940102+0000","last_undegraded":"2026-02-18T06:35:30.940102+0000","last_fullsized":"2026-02-18T06:35:30.940102+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939882+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T06:35:30.939882+0000","last_peered":"2026-02-18T06:35:30.939882+0000","last_clean":"2026-02-18T06:35:30.939882+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T06:35:30.939882+0000","last_undegraded":"2026-02-18T06:35:30.939882+0000","last_fullsized":"2026-02-18T06:35:30.939882+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.941644+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T06:35:30.941644+0000","last_peered":"2026-02-18T06:35:30.941644+0000","last_clean":"2026-02-18T06:35:30.941644+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T06:35:30.941644+0000","last_undegraded":"2026-02-18T06:35:30.941644+0000","last_fullsized":"2026-02-18T06:35:30.941644+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:31.440694+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T06:35:31.440694+0000","last_peered":"2026-02-18T06:35:31.440694+0000","last_clean":"2026-02-18T06:35:31.440694+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T06:35:31.440694+0000","last_undegraded":"2026-02-18T06:35:31.440694+0000","last_fullsized":"2026-02-18T06:35:31.440694+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:31.440672+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T06:35:31.440672+0000","last_peered":"2026-02-18T06:35:31.440672+0000","last_clean":"2026-02-18T06:35:31.440672+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T06:35:31.440672+0000","last_undegraded":"2026-02-18T06:35:31.440672+0000","last_fullsized":"2026-02-18T06:35:31.440672+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940053+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T06:35:30.940053+0000","last_peered":"2026-02-18T06:35:30.940053+0000","last_clean":"2026-02-18T06:35:30.940053+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T06:35:30.940053+0000","last_undegraded":"2026-02-18T06:35:30.940053+0000","last_fullsized":"2026-02-18T06:35:30.940053+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939451+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T06:35:30.939451+0000","last_peered":"2026-02-18T06:35:30.939451+0000","last_clean":"2026-02-18T06:35:30.939451+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T06:35:30.939451+0000","last_undegraded":"2026-02-18T06:35:30.939451+0000","last_fullsized":"2026-02-18T06:35:30.939451+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27140,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344700,"statfs":{"total":96636764160,"available":96608972800,"internally_reserved":0,"allocated":184320,"data_stored":625803,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":184320,"data_stored":625803,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8123,"internal_metadata":27582533},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T06:35:34.748 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T06:35:34.927 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:34.928 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T06:35:34.936 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-02-18T06:35:34.916758+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81368,"kb_used_data":472,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283034152,"statfs":{"total":289910292480,"available":289826971648,"internally_reserved":0,"allocated":483328,"data_stored":1283458,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":23723,"internal_metadata":82748245},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.978163"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940363+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T06:35:30.940363+0000","last_peered":"2026-02-18T06:35:30.940363+0000","last_clean":"2026-02-18T06:35:30.940363+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T06:35:30.940363+0000","last_undegraded":"2026-02-18T06:35:30.940363+0000","last_fullsized":"2026-02-18T06:35:30.940363+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939834+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T06:35:30.939834+0000","last_peered":"2026-02-18T06:35:30.939834+0000","last_clean":"2026-02-18T06:35:30.939834+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T06:35:30.939834+0000","last_undegraded":"2026-02-18T06:35:30.939834+0000","last_fullsized":"2026-02-18T06:35:30.939834+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940102+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T06:35:30.940102+0000","last_peered":"2026-02-18T06:35:30.940102+0000","last_clean":"2026-02-18T06:35:30.940102+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T06:35:30.940102+0000","last_undegraded":"2026-02-18T06:35:30.940102+0000","last_fullsized":"2026-02-18T06:35:30.940102+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939882+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T06:35:30.939882+0000","last_peered":"2026-02-18T06:35:30.939882+0000","last_clean":"2026-02-18T06:35:30.939882+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T06:35:30.939882+0000","last_undegraded":"2026-02-18T06:35:30.939882+0000","last_fullsized":"2026-02-18T06:35:30.939882+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.941644+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T06:35:30.941644+0000","last_peered":"2026-02-18T06:35:30.941644+0000","last_clean":"2026-02-18T06:35:30.941644+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T06:35:30.941644+0000","last_undegraded":"2026-02-18T06:35:30.941644+0000","last_fullsized":"2026-02-18T06:35:30.941644+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:31.440694+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T06:35:31.440694+0000","last_peered":"2026-02-18T06:35:31.440694+0000","last_clean":"2026-02-18T06:35:31.440694+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T06:35:31.440694+0000","last_undegraded":"2026-02-18T06:35:31.440694+0000","last_fullsized":"2026-02-18T06:35:31.440694+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:31.440672+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T06:35:31.440672+0000","last_peered":"2026-02-18T06:35:31.440672+0000","last_clean":"2026-02-18T06:35:31.440672+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T06:35:31.440672+0000","last_undegraded":"2026-02-18T06:35:31.440672+0000","last_fullsized":"2026-02-18T06:35:31.440672+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.940053+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T06:35:30.940053+0000","last_peered":"2026-02-18T06:35:30.940053+0000","last_clean":"2026-02-18T06:35:30.940053+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T06:35:30.940053+0000","last_undegraded":"2026-02-18T06:35:30.940053+0000","last_fullsized":"2026-02-18T06:35:30.940053+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-18T06:35:30.939451+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T06:35:30.939451+0000","last_peered":"2026-02-18T06:35:30.939451+0000","last_clean":"2026-02-18T06:35:30.939451+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T06:35:30.939451+0000","last_undegraded":"2026-02-18T06:35:30.939451+0000","last_fullsized":"2026-02-18T06:35:30.939451+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27072,"kb_used_data":112,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344768,"statfs":{"total":96636764160,"available":96609042432,"internally_reserved":0,"allocated":114688,"data_stored":31852,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27140,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344700,"statfs":{"total":96636764160,"available":96608972800,"internally_reserved":0,"allocated":184320,"data_stored":625803,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":180,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":184320,"data_stored":625803,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":8123,"internal_metadata":27582533},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T06:35:34.938 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-18T06:35:34.938 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-18T06:35:34.938 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-18T06:35:34.938 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-18T06:35:35.148 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T06:35:35.148 INFO:teuthology.orchestra.run.trial051.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-18T06:35:35.158 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-18T06:35:35.158 INFO:teuthology.run_tasks:Running task exec... 2026-02-18T06:35:35.171 INFO:teuthology.task.exec:Executing custom commands... 2026-02-18T06:35:35.171 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial051.front.sepia.ceph.com 2026-02-18T06:35:35.171 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:35.398 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror] 2026-02-18T06:35:35.398 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCKXZVpofRdLxAAMfMCb1jusw4Rmzbjq+tHTA== 2026-02-18T06:35:35.398 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-18T06:35:35.398 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:35.399 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror 2026-02-18T06:35:35.408 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:35.643 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.0] 2026-02-18T06:35:35.643 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCKXZVpOGR1NBAAKFpW8+YScStjSxOxmgH2DA== 2026-02-18T06:35:35.643 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:35.644 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:35.644 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.0 2026-02-18T06:35:35.653 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:35.878 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.1] 2026-02-18T06:35:35.878 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCKXZVpx1iDORAAhzF085iSzdJ8KuqK+NsWoA== 2026-02-18T06:35:35.878 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:35.878 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:35.879 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.1 2026-02-18T06:35:35.888 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:36.119 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.2] 2026-02-18T06:35:36.119 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCLXZVph1f1AhAAFUIxmhCwAZfQr/nC2s3F6Q== 2026-02-18T06:35:36.120 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:36.120 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:36.120 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.2 2026-02-18T06:35:36.129 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:36.361 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.3] 2026-02-18T06:35:36.361 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCLXZVp2ODzBxAA+WpUM+cgvq0gFI3u3J/25w== 2026-02-18T06:35:36.361 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:36.362 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:36.362 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.3 2026-02-18T06:35:36.372 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:36.603 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror] 2026-02-18T06:35:36.603 INFO:teuthology.orchestra.run.trial051.stdout: key = AQCgXZVpQxTVNxAAbDDSb26swyUDnjrF0TrH1A== 2026-02-18T06:35:36.603 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-18T06:35:36.603 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:36.603 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror 2026-02-18T06:35:36.614 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:36.843 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.0] 2026-02-18T06:35:36.843 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpkl5CARAANlcXg7ztyKjiNRyV0kpqHg== 2026-02-18T06:35:36.843 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:36.843 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:36.843 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.0 2026-02-18T06:35:36.853 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:37.085 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.1] 2026-02-18T06:35:37.085 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVp/r9cBhAA77Ii1SOaRxahWR7NyRGfrA== 2026-02-18T06:35:37.085 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:37.086 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:37.086 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.1 2026-02-18T06:35:37.095 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:37.331 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.2] 2026-02-18T06:35:37.331 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVp0phoCxAAyM4Ct43JguV9lgb0RXIuVQ== 2026-02-18T06:35:37.331 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:37.332 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:37.332 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.2 2026-02-18T06:35:37.342 DEBUG:teuthology.orchestra.run.trial051:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-18T06:35:37.569 INFO:teuthology.orchestra.run.trial051.stdout:[client.mirror.3] 2026-02-18T06:35:37.569 INFO:teuthology.orchestra.run.trial051.stdout: key = AQChXZVpnNmGEBAAe2O1FtE8VHCbVc9SiMSH0w== 2026-02-18T06:35:37.569 INFO:teuthology.orchestra.run.trial051.stdout: caps mon = "profile rbd-mirror" 2026-02-18T06:35:37.569 INFO:teuthology.orchestra.run.trial051.stdout: caps osd = "profile rbd" 2026-02-18T06:35:37.569 INFO:teuthology.orchestra.run.trial051.stderr:updated caps for client.mirror.3 2026-02-18T06:35:37.580 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T06:35:37.601 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T06:35:37.604 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T06:35:37.604 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T06:35:37.620 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T06:35:37.622 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T06:35:37.622 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.637 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T06:35:37.637 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T06:35:37.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T06:35:37.640 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.655 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-18T06:35:37.655 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-18T06:35:37.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-18T06:35:37.658 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.673 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:35:37.674 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:35:37.676 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:35:37.676 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.692 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T06:35:37.692 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T06:35:37.695 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T06:35:37.695 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.711 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T06:35:37.711 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T06:35:37.714 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T06:35:37.714 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-18T06:35:37.729 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-18T06:35:37.730 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-18T06:35:37.732 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-18T06:35:37.732 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-18T06:35:37.752 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771396537 2026-02-18T06:35:37.752 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-18T06:35:37.752 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-18T06:35:37.766 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771396537 2026-02-18T06:35:37.767 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-18T06:35:37.767 INFO:teuthology.run_tasks:Running task workunit... 2026-02-18T06:35:37.790 INFO:tasks.workunit:Pulling workunits from ref adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:35:37.791 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-18T06:35:37.791 DEBUG:teuthology.orchestra.run.trial051:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T06:35:37.791 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-18T06:35:37.791 INFO:tasks.rbd_mirror_thrash:waiting for 70.54072324975134 secs before thrashing 2026-02-18T06:35:37.791 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-18T06:35:37.792 INFO:tasks.rbd_mirror_thrash:waiting for 85.54443216947058 secs before thrashing 2026-02-18T06:35:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T06:35:37.807 INFO:teuthology.orchestra.run.trial051.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-18T06:35:37.807 DEBUG:teuthology.orchestra.run.trial051:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T06:35:37.865 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T06:35:37.866 DEBUG:teuthology.orchestra.run.trial051:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-18T06:35:37.920 INFO:tasks.workunit:timeout=6h 2026-02-18T06:35:37.921 INFO:tasks.workunit:cleanup=True 2026-02-18T06:35:37.921 DEBUG:teuthology.orchestra.run.trial051:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout adc1923395f2bcd6ae89b5d331d278364d1848ed 2026-02-18T06:35:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Note: switching to 'adc1923395f2bcd6ae89b5d331d278364d1848ed'. 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:state without impacting any branches by switching back to a branch. 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: git switch -c 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Or undo this operation with: 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: git switch - 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:35:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:HEAD is now at adc1923395f Merge PR #67279 into tentacle 2026-02-18T06:35:57.729 DEBUG:teuthology.orchestra.run.trial051:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-18T06:35:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-18T06:35:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-18T06:35:57.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-18T06:35:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-18T06:35:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-18T06:35:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-18T06:35:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-18T06:35:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-18T06:35:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-18T06:35:57.966 DEBUG:teuthology.orchestra.run.trial051:> set -ex 2026-02-18T06:35:57.967 DEBUG:teuthology.orchestra.run.trial051:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-18T06:35:58.022 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-18T06:35:58.023 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-18T06:35:58.023 DEBUG:teuthology.orchestra.run.trial051:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=adc1923395f2bcd6ae89b5d331d278364d1848ed TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T06:35:58.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ IMAGE_COUNT=50 2026-02-18T06:35:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ export LOCKDEP=0 2026-02-18T06:35:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ LOCKDEP=0 2026-02-18T06:35:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CLUSTER1=cluster1 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CLUSTER2=cluster2 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ POOL=mirror 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ PARENT_POOL=mirror_parent 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ NS1=ns1 2026-02-18T06:35:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ NS2=ns2 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ TEMPDIR= 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ID=mirror 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ RBD_IMAGE_FEATURES=layering 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ MIRROR_POOL_MODE=image 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' snapshot = snapshot ']' 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ MIRROR_POOL_MODE=image 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS='--id mirror' 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-18T06:35:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-18T06:35:58.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-18T06:35:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-18T06:35:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_BIN=. 2026-02-18T06:35:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_SRC=. 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 -gt 0 ']' 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ setup 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local c 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ setup_tempdir 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mktemp -d 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ TEMPDIR=/tmp/tmp.y3X31uzVRt 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z 1 ']' 2026-02-18T06:35:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ setup_pools cluster1 cluster2 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster2 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mon_map_file 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mon_addr 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local admin_key_file 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:35:58.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-18T06:35:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror' already exists 2026-02-18T06:35:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:35:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-18T06:35:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror_parent' already exists 2026-02-18T06:35:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:35:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-18T06:36:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-18T06:36:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2026-02-18T06:36:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-18T06:36:05.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-18T06:36:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-18T06:36:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-18T06:36:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-18T06:36:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-18T06:36:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z ']' 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z ']' 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_add cluster1 mirror cluster2 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local client_cluster=cluster2 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster2 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid_var_name 2026-02-18T06:36:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local error_code 2026-02-18T06:36:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local peer_uuid 2026-02-18T06:36:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T06:36:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:36:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-18T06:36:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_uuid=eb1a4512-33b4-4b2d-956c-d0c2b523237e 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 -eq 0 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror_parent 2026-02-18T06:36:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local client_cluster=cluster2 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster2 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid_var_name 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local error_code 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local peer_uuid 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:36:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-18T06:36:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_uuid=b400b2e9-b653-47da-be9b-a4b5eff4034e 2026-02-18T06:36:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 -eq 0 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ setup_pools cluster2 cluster1 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster1 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mon_map_file 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mon_addr 2026-02-18T06:36:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local admin_key_file 2026-02-18T06:36:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid 2026-02-18T06:36:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-18T06:36:06.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror' already exists 2026-02-18T06:36:06.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:06.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-18T06:36:07.894 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:36:07.891+0000 7f94837b7a40 -1 rbd::mirror::PoolReplayer: 0x558df0050c00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:07.894 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:36:07.891+0000 7f37508b7a40 -1 rbd::mirror::PoolReplayer: 0x5623b98aec00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:07.895 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:36:07.891+0000 7f3ffb315a40 -1 rbd::mirror::PoolReplayer: 0x5625e092c900 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:07.895 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:36:07.891+0000 7efdd99bca40 -1 rbd::mirror::PoolReplayer: 0x55751243e900 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror_parent' already exists 2026-02-18T06:36:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-18T06:36:08.888 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:36:08.885+0000 7efdd99bca40 -1 rbd::mirror::PoolReplayer: 0x557513201500 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:08.888 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:36:08.885+0000 7f37508b7a40 -1 rbd::mirror::PoolReplayer: 0x5623b99fb800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:08.889 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:36:08.885+0000 7f3ffb315a40 -1 rbd::mirror::PoolReplayer: 0x5625e16e7500 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:08.890 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:36:08.887+0000 7f94837b7a40 -1 rbd::mirror::PoolReplayer: 0x558df0e21800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-18T06:36:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-18T06:36:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T06:36:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2026-02-18T06:36:13.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-18T06:36:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-18T06:36:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-18T06:36:14.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-18T06:36:14.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-18T06:36:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-18T06:36:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-18T06:36:14.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z ']' 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z ']' 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_add cluster2 mirror cluster1 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local client_cluster=cluster1 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster1 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid_var_name 2026-02-18T06:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local error_code 2026-02-18T06:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local peer_uuid 2026-02-18T06:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T06:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_uuid=65088dbf-1be6-4ca5-8c8e-c0ed9090a488 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 -eq 0 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-18T06:36:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror_parent 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local client_cluster=cluster1 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_cluster=cluster1 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local uuid_var_name 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local error_code 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local peer_uuid 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 16 32 2026-02-18T06:36:14.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:36:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ peer_uuid=d5f48a1f-25fa-416b-b429-35793738d811 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 17 ']' 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 -eq 0 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirrors cluster1 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 0 3 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster1:0 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:0 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster1:0 2026-02-18T06:36:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster1 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=0 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster1 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=0 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster1:1 2026-02-18T06:36:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster1:1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=1 2026-02-18T06:36:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster1:2 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:2 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster1:2 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster1 2026-02-18T06:36:14.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=2 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster1 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=2 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster1:3 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:3 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster1:3 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster1 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=3 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster1 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=3 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirrors cluster2 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 0 3 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster2:0 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:0 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster2:0 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster2 2026-02-18T06:36:14.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=0 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster2 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster2 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=0 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster2:1 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:1 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-18T06:36:14.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster2:1 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster2 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=1 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster2 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster2 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=1 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster2:2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster2:2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=2 2026-02-18T06:36:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster2:3 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:3 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster2:3 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster2 2026-02-18T06:36:14.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=3 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster2 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster2 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=3 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 06:36:14 TEST: add image and test replay after client crashes 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 06:36:14 TEST: add image and test replay after client crashes 2026-02-18T06:36:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=test 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror test 512M 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:36:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/test 2026-02-18T06:36:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:36:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:36:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:36:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-18T06:36:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:36:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-18T06:36:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10c823500987 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10c823500987 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:36:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:14 2026 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:36:14 2026 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:14 2026 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:36:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:36:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:36:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:36:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:36:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:36:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:36:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:36:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:36:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:36:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:16.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:16.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:36:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:36:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:36:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:36:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:36:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T06:36:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:16.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 1 2026-02-18T06:36:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:17.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:17.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:36:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:36:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:36:17.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:17.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:36:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:36:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:36:17.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:17.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:17.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:17.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:36:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T06:36:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:17.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:36:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:36:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:17.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T06:36:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 2 2026-02-18T06:36:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:19.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:19.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:19.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:36:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:36:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:36:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:36:19.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:36:19.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:36:19.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:19.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:19.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:19.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T06:36:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 4 2026-02-18T06:36:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:23.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:23.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:23.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:23.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:36:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:36:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:36:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:23.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:23.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:36:23.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:36:23.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:36:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:36:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T06:36:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:36:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:36:23.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:23.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:36:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:36:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:36:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T06:36:23.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:23.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:23.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 8 2026-02-18T06:36:31.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:31.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:31.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:31.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:31.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:31.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:31.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:36:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:36:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:36:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:36:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:36:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:36:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:32.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:36:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:36:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:36:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:36:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:36:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:36:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-18T06:36:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 8 2026-02-18T06:36:37.892 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:36:37.890+0000 7efdd99bca40 -1 rbd::mirror::Mirror: 0x55751156a240 update_pool_replayers: restarting failed pool replayer for uuid: eb1a4512-33b4-4b2d-956c-d0c2b523237e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:36:37.890+0000 7f37508b7a40 -1 rbd::mirror::Mirror: 0x5623b89c0240 update_pool_replayers: restarting failed pool replayer for uuid: eb1a4512-33b4-4b2d-956c-d0c2b523237e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.893 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:36:37.890+0000 7f3ffb315a40 -1 rbd::mirror::Mirror: 0x5625dfa62240 update_pool_replayers: restarting failed pool replayer for uuid: eb1a4512-33b4-4b2d-956c-d0c2b523237e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.894 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:36:37.891+0000 7f94837b7a40 -1 rbd::mirror::Mirror: 0x558def1a4240 update_pool_replayers: restarting failed pool replayer for uuid: eb1a4512-33b4-4b2d-956c-d0c2b523237e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:36:37.909+0000 7f37508b7a40 -1 rbd::mirror::Mirror: 0x5623b89c0240 update_pool_replayers: restarting failed pool replayer for uuid: b400b2e9-b653-47da-be9b-a4b5eff4034e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:36:37.910+0000 7f3ffb315a40 -1 rbd::mirror::Mirror: 0x5625dfa62240 update_pool_replayers: restarting failed pool replayer for uuid: b400b2e9-b653-47da-be9b-a4b5eff4034e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.916 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:36:37.914+0000 7efdd99bca40 -1 rbd::mirror::Mirror: 0x55751156a240 update_pool_replayers: restarting failed pool replayer for uuid: b400b2e9-b653-47da-be9b-a4b5eff4034e cluster: cluster2 client: client.mirror 2026-02-18T06:36:37.917 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:36:37.914+0000 7f94837b7a40 -1 rbd::mirror::Mirror: 0x558def1a4240 update_pool_replayers: restarting failed pool replayer for uuid: b400b2e9-b653-47da-be9b-a4b5eff4034e cluster: cluster2 client: client.mirror 2026-02-18T06:36:40.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:36:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:36:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:36:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:36:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:36:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:36:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:36:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:36:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:36:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:36:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:36:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:36:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:36:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:36:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:36:40.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:36:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:36:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:36:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:36:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:36:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:36:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:36:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:36:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:36:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:36:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:36:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:36:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name= 2026-02-18T06:36:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 10 2026-02-18T06:36:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:36:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:36:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:40.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:36:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:36:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:36:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:36:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 67269632 bytes 2026-02-18T06:36:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 105889792 bytes 2026-02-18T06:36:40.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 116299776 bytes 2026-02-18T06:36:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 195515392 bytes 2026-02-18T06:36:40.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 220700672 bytes 2026-02-18T06:36:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 242405376 bytes 2026-02-18T06:36:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 285422592 bytes 2026-02-18T06:36:40.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 352657408 bytes 2026-02-18T06:36:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 440059904 bytes 2026-02-18T06:36:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 529521664 bytes 2026-02-18T06:36:40.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 561509376 bytes 2026-02-18T06:36:40.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 604013568 bytes 2026-02-18T06:36:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 644166656 bytes 2026-02-18T06:36:40.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 677582848 bytes 2026-02-18T06:36:40.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 695476224 bytes 2026-02-18T06:36:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 737492992 bytes 2026-02-18T06:36:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 796150784 bytes 2026-02-18T06:36:40.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 820206592 bytes 2026-02-18T06:36:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 833405952 bytes 2026-02-18T06:36:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 874491904 bytes 2026-02-18T06:36:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 885424128 bytes 2026-02-18T06:36:41.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 930877440 bytes 2026-02-18T06:36:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 956285952 bytes 2026-02-18T06:36:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1048845312 bytes 2026-02-18T06:36:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1093594112 bytes 2026-02-18T06:36:41.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1123893248 bytes 2026-02-18T06:36:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1147627520 bytes 2026-02-18T06:36:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1166006272 bytes 2026-02-18T06:36:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1174765568 bytes 2026-02-18T06:36:41.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1229598720 bytes 2026-02-18T06:36:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1267261440 bytes 2026-02-18T06:36:41.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1325582336 bytes 2026-02-18T06:36:41.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1370390528 bytes 2026-02-18T06:36:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1404676096 bytes 2026-02-18T06:36:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1469733888 bytes 2026-02-18T06:36:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1528452096 bytes 2026-02-18T06:36:41.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1608802304 bytes 2026-02-18T06:36:41.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1672735744 bytes 2026-02-18T06:36:41.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1692291072 bytes 2026-02-18T06:36:41.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1747558400 bytes 2026-02-18T06:36:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1779259392 bytes 2026-02-18T06:36:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1840599040 bytes 2026-02-18T06:36:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1890996224 bytes 2026-02-18T06:36:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1960205312 bytes 2026-02-18T06:36:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2002964480 bytes 2026-02-18T06:36:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2059982848 bytes 2026-02-18T06:36:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2178403328 bytes 2026-02-18T06:36:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2201697280 bytes 2026-02-18T06:36:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2253998080 bytes 2026-02-18T06:36:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2322154496 bytes 2026-02-18T06:36:42.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2354236416 bytes 2026-02-18T06:36:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2382008320 bytes 2026-02-18T06:36:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2412102656 bytes 2026-02-18T06:36:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2412733440 bytes 2026-02-18T06:36:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2449704960 bytes 2026-02-18T06:36:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2534669312 bytes 2026-02-18T06:36:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2555142144 bytes 2026-02-18T06:36:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2653945856 bytes 2026-02-18T06:36:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2676254720 bytes 2026-02-18T06:36:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2722597888 bytes 2026-02-18T06:36:42.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2774018048 bytes 2026-02-18T06:36:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2868504576 bytes 2026-02-18T06:36:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2932641792 bytes 2026-02-18T06:36:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3000042496 bytes 2026-02-18T06:36:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3048056832 bytes 2026-02-18T06:36:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3116568576 bytes 2026-02-18T06:36:43.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3184504832 bytes 2026-02-18T06:36:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3185131520 bytes 2026-02-18T06:36:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3203929088 bytes 2026-02-18T06:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3277996032 bytes 2026-02-18T06:36:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3304846336 bytes 2026-02-18T06:36:43.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3332337664 bytes 2026-02-18T06:36:43.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3346694144 bytes 2026-02-18T06:36:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3421475840 bytes 2026-02-18T06:36:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3507035136 bytes 2026-02-18T06:36:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3557600256 bytes 2026-02-18T06:36:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3582052352 bytes 2026-02-18T06:36:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3627508736 bytes 2026-02-18T06:36:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3757417472 bytes 2026-02-18T06:36:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3791643648 bytes 2026-02-18T06:36:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3830841344 bytes 2026-02-18T06:36:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3882778624 bytes 2026-02-18T06:36:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3928251392 bytes 2026-02-18T06:36:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3992049664 bytes 2026-02-18T06:36:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 4073840640 bytes 2026-02-18T06:36:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4102700032 bytes 2026-02-18T06:36:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4125408256 bytes 2026-02-18T06:36:44.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4155348992 bytes 2026-02-18T06:36:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4205575168 bytes 2026-02-18T06:36:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4229966848 bytes 2026-02-18T06:36:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4264513536 bytes 2026-02-18T06:36:44.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4311961600 bytes 2026-02-18T06:36:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4334524416 bytes 2026-02-18T06:36:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4343533568 bytes 2026-02-18T06:36:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4409559040 bytes 2026-02-18T06:36:44.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4412577792 bytes 2026-02-18T06:36:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4490449920 bytes 2026-02-18T06:36:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4523534336 bytes 2026-02-18T06:36:44.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4531537920 bytes 2026-02-18T06:36:44.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4567401472 bytes 2026-02-18T06:36:44.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4578992128 bytes 2026-02-18T06:36:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4593876992 bytes 2026-02-18T06:36:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4611443712 bytes 2026-02-18T06:36:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4659361792 bytes 2026-02-18T06:36:44.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4669002752 bytes 2026-02-18T06:36:44.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4731397120 bytes 2026-02-18T06:36:44.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4773049344 bytes 2026-02-18T06:36:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4799806464 bytes 2026-02-18T06:36:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4849486848 bytes 2026-02-18T06:36:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4850089984 bytes 2026-02-18T06:36:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4903556096 bytes 2026-02-18T06:36:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4985177088 bytes 2026-02-18T06:36:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5070510080 bytes 2026-02-18T06:36:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5109563392 bytes 2026-02-18T06:36:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5123120128 bytes 2026-02-18T06:36:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5140686848 bytes 2026-02-18T06:36:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5159143424 bytes 2026-02-18T06:36:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5237867520 bytes 2026-02-18T06:36:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5319145472 bytes 2026-02-18T06:36:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5367086080 bytes 2026-02-18T06:36:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5418933248 bytes 2026-02-18T06:36:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5437976576 bytes 2026-02-18T06:36:45.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5444197376 bytes 2026-02-18T06:36:45.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5528469504 bytes 2026-02-18T06:36:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5579058176 bytes 2026-02-18T06:36:45.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5609969664 bytes 2026-02-18T06:36:45.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5627695104 bytes 2026-02-18T06:36:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5628310528 bytes 2026-02-18T06:36:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5628939264 bytes 2026-02-18T06:36:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5697234944 bytes 2026-02-18T06:36:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5720425472 bytes 2026-02-18T06:36:45.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5776547840 bytes 2026-02-18T06:36:45.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5873775616 bytes 2026-02-18T06:36:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5927997440 bytes 2026-02-18T06:36:45.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5973324800 bytes 2026-02-18T06:36:45.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5973965824 bytes 2026-02-18T06:36:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6042148864 bytes 2026-02-18T06:36:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6042771456 bytes 2026-02-18T06:36:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6067449856 bytes 2026-02-18T06:36:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6083614720 bytes 2026-02-18T06:36:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6103409664 bytes 2026-02-18T06:36:46.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6132232192 bytes 2026-02-18T06:36:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6132842496 bytes 2026-02-18T06:36:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6147676160 bytes 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:36:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:36:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:36:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:36:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:36:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:36:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:36:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:36:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"} 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:36:39 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:36:39' 2026-02-18T06:36:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:36:47.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:36:47.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:36:47.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"} 2026-02-18T06:36:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:36:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:36:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:36:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:36:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:36:47.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:36:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:36:47.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:36:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:36:39' 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:36:39' 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:36:39' 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:36:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:36:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142024282mirror.0trial05120.2.0-670-gadc19233falseOK42054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK42134281mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:36:39cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:36:39' 2026-02-18T06:36:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:36:47.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:36:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:36:39' 2026-02-18T06:36:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:36:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:36:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:36:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:36:47.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap1 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap1 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:36:47.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:36:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1 2026-02-18T06:36:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:36:48.332 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T06:36:48.333 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T06:36:48.333 INFO:tasks.rbd_mirror_thrash:waiting for 9.40200760949178 secs before reviving daemons 2026-02-18T06:36:48.333 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:36:48.332+0000 7f9480757640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 29880) UID: 1000 2026-02-18T06:36:57.736 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T06:36:57.736 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:36:57.736 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T06:36:57.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T06:36:57.737 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T06:36:57.740 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T06:36:57.740 INFO:tasks.rbd_mirror_thrash:waiting for 75.32593359559067 secs before thrashing 2026-02-18T06:37:03.337 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.38982984376841767) > weight (0.25) 2026-02-18T06:37:03.338 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-18T06:37:03.338 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-18T06:37:03.338 INFO:tasks.rbd_mirror_thrash:waiting for 5.991773512691083 secs before reviving daemons 2026-02-18T06:37:03.339 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T06:37:03.337+0000 7f6ff361d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 30070) UID: 1000 2026-02-18T06:37:09.331 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-18T06:37:09.331 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:37:09.331 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-18T06:37:09.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-18T06:37:09.332 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-18T06:37:09.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-18T06:37:09.335 INFO:tasks.rbd_mirror_thrash:waiting for 106.42096451495789 secs before thrashing 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:37:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:16.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:37:16.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:37:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:37:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:37:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:37:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:37:16.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:37:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:37:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:37:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:37:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:37:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:37:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:37:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:37:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:37:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:37:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:37:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:37:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:37:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:37:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:37:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:37:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:37:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:37:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:37:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:37:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:37:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:37:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:37:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:37:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:37:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 5 2026-02-18T06:37:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:37:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:37:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:37:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:37:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:37:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:37:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:37:17.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:37:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:37:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=5 2026-02-18T06:37:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:37:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:37:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:37:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:37:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:37:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=3 2026-02-18T06:37:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 5 = 3 2026-02-18T06:37:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:37:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:37:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:37:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:37:18.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:37:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=3 2026-02-18T06:37:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 5 = 3 2026-02-18T06:37:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:37:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:37:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:37:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:37:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:37:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:37:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:37:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:37:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:37:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 5 = '' 2026-02-18T06:37:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:37:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:37:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:37:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:37:21.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:37:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=5 2026-02-18T06:37:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 5 = 5 2026-02-18T06:37:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-18T06:37:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:37:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:37:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:37:22.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap1 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:37:22.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:37:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:37:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:37:22.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:37:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:37:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:37:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap1-clean 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1-clean 2026-02-18T06:37:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:37:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:37:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:37:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:37:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T06:37:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:37:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:37:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 25755648 bytes 2026-02-18T06:37:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 109491200 bytes 2026-02-18T06:37:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 155347968 bytes 2026-02-18T06:37:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 155972608 bytes 2026-02-18T06:37:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 239666176 bytes 2026-02-18T06:37:24.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 310693888 bytes 2026-02-18T06:37:24.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 349998080 bytes 2026-02-18T06:37:24.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 405916672 bytes 2026-02-18T06:37:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 441947136 bytes 2026-02-18T06:37:24.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 497267712 bytes 2026-02-18T06:37:24.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 506127360 bytes 2026-02-18T06:37:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 588093440 bytes 2026-02-18T06:37:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 590319616 bytes 2026-02-18T06:37:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 626522112 bytes 2026-02-18T06:37:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 695660544 bytes 2026-02-18T06:37:24.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 728769536 bytes 2026-02-18T06:37:24.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 823218176 bytes 2026-02-18T06:37:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 889171968 bytes 2026-02-18T06:37:24.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 922248192 bytes 2026-02-18T06:37:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 965298176 bytes 2026-02-18T06:37:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1027244032 bytes 2026-02-18T06:37:24.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1055920128 bytes 2026-02-18T06:37:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1056545792 bytes 2026-02-18T06:37:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1061305344 bytes 2026-02-18T06:37:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1101458432 bytes 2026-02-18T06:37:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1117476864 bytes 2026-02-18T06:37:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1169638400 bytes 2026-02-18T06:37:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1186181120 bytes 2026-02-18T06:37:25.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1195811840 bytes 2026-02-18T06:37:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1218938880 bytes 2026-02-18T06:37:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1232754688 bytes 2026-02-18T06:37:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1242880000 bytes 2026-02-18T06:37:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1243510784 bytes 2026-02-18T06:37:25.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1332825088 bytes 2026-02-18T06:37:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1338400768 bytes 2026-02-18T06:37:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1413679104 bytes 2026-02-18T06:37:25.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1444274176 bytes 2026-02-18T06:37:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1515831296 bytes 2026-02-18T06:37:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1547419648 bytes 2026-02-18T06:37:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1588798464 bytes 2026-02-18T06:37:25.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1613961216 bytes 2026-02-18T06:37:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1616556032 bytes 2026-02-18T06:37:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1668622336 bytes 2026-02-18T06:37:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1704270848 bytes 2026-02-18T06:37:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1729999872 bytes 2026-02-18T06:37:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1801726976 bytes 2026-02-18T06:37:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1849127936 bytes 2026-02-18T06:37:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1960925184 bytes 2026-02-18T06:37:25.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1975725056 bytes 2026-02-18T06:37:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2031833088 bytes 2026-02-18T06:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2069076992 bytes 2026-02-18T06:37:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2085091328 bytes 2026-02-18T06:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2139474944 bytes 2026-02-18T06:37:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2165967872 bytes 2026-02-18T06:37:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2233757696 bytes 2026-02-18T06:37:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2296504320 bytes 2026-02-18T06:37:26.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2303706112 bytes 2026-02-18T06:37:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2347366400 bytes 2026-02-18T06:37:26.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2412571648 bytes 2026-02-18T06:37:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2504142848 bytes 2026-02-18T06:37:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2537864192 bytes 2026-02-18T06:37:26.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2541936640 bytes 2026-02-18T06:37:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2596272128 bytes 2026-02-18T06:37:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2621105152 bytes 2026-02-18T06:37:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2632953856 bytes 2026-02-18T06:37:26.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2659931136 bytes 2026-02-18T06:37:26.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2711470080 bytes 2026-02-18T06:37:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2749030400 bytes 2026-02-18T06:37:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2782300160 bytes 2026-02-18T06:37:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2833519616 bytes 2026-02-18T06:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2878033920 bytes 2026-02-18T06:37:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2913434624 bytes 2026-02-18T06:37:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2977119232 bytes 2026-02-18T06:37:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3022372864 bytes 2026-02-18T06:37:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3043024896 bytes 2026-02-18T06:37:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3059192832 bytes 2026-02-18T06:37:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3093495808 bytes 2026-02-18T06:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3132789760 bytes 2026-02-18T06:37:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3213727744 bytes 2026-02-18T06:37:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3242554368 bytes 2026-02-18T06:37:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3318173696 bytes 2026-02-18T06:37:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3369971712 bytes 2026-02-18T06:37:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3382564864 bytes 2026-02-18T06:37:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3453750272 bytes 2026-02-18T06:37:27.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3501224960 bytes 2026-02-18T06:37:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3574695936 bytes 2026-02-18T06:37:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3679641600 bytes 2026-02-18T06:37:27.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3764339712 bytes 2026-02-18T06:37:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3822315520 bytes 2026-02-18T06:37:27.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3845771264 bytes 2026-02-18T06:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3846366208 bytes 2026-02-18T06:37:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3869403136 bytes 2026-02-18T06:37:27.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3944707072 bytes 2026-02-18T06:37:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3951328256 bytes 2026-02-18T06:37:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4002402304 bytes 2026-02-18T06:37:28.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4023101440 bytes 2026-02-18T06:37:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4060340224 bytes 2026-02-18T06:37:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4088099840 bytes 2026-02-18T06:37:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4125004800 bytes 2026-02-18T06:37:28.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4183456768 bytes 2026-02-18T06:37:28.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4234400768 bytes 2026-02-18T06:37:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4287444992 bytes 2026-02-18T06:37:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4294895616 bytes 2026-02-18T06:37:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4321943552 bytes 2026-02-18T06:37:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4354570240 bytes 2026-02-18T06:37:28.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4358298624 bytes 2026-02-18T06:37:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4408467456 bytes 2026-02-18T06:37:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4438141952 bytes 2026-02-18T06:37:28.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4485554176 bytes 2026-02-18T06:37:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4497527808 bytes 2026-02-18T06:37:28.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4508210176 bytes 2026-02-18T06:37:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4528464896 bytes 2026-02-18T06:37:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4529070080 bytes 2026-02-18T06:37:28.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4576200704 bytes 2026-02-18T06:37:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4664336384 bytes 2026-02-18T06:37:28.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4697519104 bytes 2026-02-18T06:37:28.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4703166464 bytes 2026-02-18T06:37:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4762066944 bytes 2026-02-18T06:37:28.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4772806656 bytes 2026-02-18T06:37:28.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4809125888 bytes 2026-02-18T06:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4829524992 bytes 2026-02-18T06:37:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4866633728 bytes 2026-02-18T06:37:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4904833024 bytes 2026-02-18T06:37:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4949444608 bytes 2026-02-18T06:37:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4994965504 bytes 2026-02-18T06:37:29.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5102738432 bytes 2026-02-18T06:37:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5150435328 bytes 2026-02-18T06:37:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5192445952 bytes 2026-02-18T06:37:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5250787328 bytes 2026-02-18T06:37:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5312778240 bytes 2026-02-18T06:37:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5362378752 bytes 2026-02-18T06:37:29.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5409703936 bytes 2026-02-18T06:37:29.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5512660992 bytes 2026-02-18T06:37:29.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5531129856 bytes 2026-02-18T06:37:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5570961408 bytes 2026-02-18T06:37:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5623790592 bytes 2026-02-18T06:37:29.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5688659968 bytes 2026-02-18T06:37:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5707619328 bytes 2026-02-18T06:37:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5753299968 bytes 2026-02-18T06:37:29.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5769964544 bytes 2026-02-18T06:37:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5772018688 bytes 2026-02-18T06:37:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5818861568 bytes 2026-02-18T06:37:29.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5886490624 bytes 2026-02-18T06:37:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5959704576 bytes 2026-02-18T06:37:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6016521216 bytes 2026-02-18T06:37:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6053269504 bytes 2026-02-18T06:37:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6110109696 bytes 2026-02-18T06:37:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6160777216 bytes 2026-02-18T06:37:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6186070016 bytes 2026-02-18T06:37:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6215018496 bytes 2026-02-18T06:37:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6291037184 bytes 2026-02-18T06:37:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6355027968 bytes 2026-02-18T06:37:30.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6375810048 bytes 2026-02-18T06:37:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6382230528 bytes 2026-02-18T06:37:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6384833536 bytes 2026-02-18T06:37:30.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6427819008 bytes 2026-02-18T06:37:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6476553216 bytes 2026-02-18T06:37:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6539926528 bytes 2026-02-18T06:37:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6580696064 bytes 2026-02-18T06:37:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6640518144 bytes 2026-02-18T06:37:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6764952576 bytes 2026-02-18T06:37:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6774710272 bytes 2026-02-18T06:37:30.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6814405632 bytes 2026-02-18T06:37:30.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6849748992 bytes 2026-02-18T06:37:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6879984640 bytes 2026-02-18T06:37:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6926729216 bytes 2026-02-18T06:37:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6946105344 bytes 2026-02-18T06:37:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6965734400 bytes 2026-02-18T06:37:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6991458304 bytes 2026-02-18T06:37:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7008660480 bytes 2026-02-18T06:37:31.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7053704192 bytes 2026-02-18T06:37:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7124787200 bytes 2026-02-18T06:37:31.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7158235136 bytes 2026-02-18T06:37:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7226049536 bytes 2026-02-18T06:37:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7281878016 bytes 2026-02-18T06:37:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7295114240 bytes 2026-02-18T06:37:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7332139008 bytes 2026-02-18T06:37:31.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7346706432 bytes 2026-02-18T06:37:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7386400768 bytes 2026-02-18T06:37:31.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7450342400 bytes 2026-02-18T06:37:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7486300160 bytes 2026-02-18T06:37:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7496341504 bytes 2026-02-18T06:37:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7526755328 bytes 2026-02-18T06:37:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7545590784 bytes 2026-02-18T06:37:31.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7600916480 bytes 2026-02-18T06:37:31.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7666630656 bytes 2026-02-18T06:37:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7673659392 bytes 2026-02-18T06:37:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7782308864 bytes 2026-02-18T06:37:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7827422208 bytes 2026-02-18T06:37:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7871387648 bytes 2026-02-18T06:37:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7900104704 bytes 2026-02-18T06:37:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7931814912 bytes 2026-02-18T06:37:32.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7967726592 bytes 2026-02-18T06:37:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7973370880 bytes 2026-02-18T06:37:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8074399744 bytes 2026-02-18T06:37:32.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8118804480 bytes 2026-02-18T06:37:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8119412736 bytes 2026-02-18T06:37:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8209831936 bytes 2026-02-18T06:37:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8249017344 bytes 2026-02-18T06:37:32.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8281765888 bytes 2026-02-18T06:37:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8401056768 bytes 2026-02-18T06:37:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8438326272 bytes 2026-02-18T06:37:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8458198016 bytes 2026-02-18T06:37:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8485054464 bytes 2026-02-18T06:37:32.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8493159424 bytes 2026-02-18T06:37:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8551568384 bytes 2026-02-18T06:37:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8564990976 bytes 2026-02-18T06:37:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8601569280 bytes 2026-02-18T06:37:32.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8633663488 bytes 2026-02-18T06:37:32.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8697332736 bytes 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:37:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:37:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:37:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:37:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:37:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396574,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"} 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:37:07 2026-02-18T06:37:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:37:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:37:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:37:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:37:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:37:08' 2026-02-18T06:37:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:37:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:37:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:37:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396574,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"} 2026-02-18T06:37:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:37:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:37:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:37:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:37:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:37:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:37:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:37:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:37:33.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:37:07' 2026-02-18T06:37:33.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:37:07' 2026-02-18T06:37:33.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:37:07' 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:37:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:37:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK42134281mirror.3trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396574,"remote_snapshot_timestamp":1771396574,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:37:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:37:08' 2026-02-18T06:37:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:37:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:37:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:37:07' 2026-02-18T06:37:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:37:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:37:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:37:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:37:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap2 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap2 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2 2026-02-18T06:37:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:38:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:38:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:38:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:38:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:38:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:38:03.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:38:03.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:38:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:38:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:38:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:38:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:38:03.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:38:03.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:38:03.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:38:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:38:03.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:38:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:38:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:38:03.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:38:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:38:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:38:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:38:03.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:38:03.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:38:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:38:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:38:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:38:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:38:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:38:03.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:38:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:38:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:38:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:38:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:38:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:38:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:38:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:38:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:38:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:38:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 8 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:38:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:38:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:38:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:38:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:38:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:38:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:38:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=8 2026-02-18T06:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:05.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:38:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:06.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:38:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:38:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:38:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:38:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 8 = '' 2026-02-18T06:38:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:06.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:38:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:38:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:38:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:38:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:38:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 8 = '' 2026-02-18T06:38:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:38:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:07.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:07.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:07.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:07.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 8 = '' 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:38:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:09.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=8 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 8 = 8 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2 2026-02-18T06:38:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:38:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:38:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:38:10.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-18T06:38:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 4 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:38:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:38:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:38:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap1-clean ']' 2026-02-18T06:38:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-18T06:38:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1-clean 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:38:10.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:38:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:38:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap2 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2 2026-02-18T06:38:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:38:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:38:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:38:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:38:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:38:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:12.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap2-clean 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2-clean 2026-02-18T06:38:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:38:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:38:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:38:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:38:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:38:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:38:12.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:38:13.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 39950336 bytes 2026-02-18T06:38:13.067 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7326754496233413) > weight (0.25) 2026-02-18T06:38:13.067 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7032983181377299) > weight (0.25) 2026-02-18T06:38:13.067 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9583652034733552) > weight (0.25) 2026-02-18T06:38:13.067 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-18T06:38:13.067 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-18T06:38:13.068 INFO:tasks.rbd_mirror_thrash:waiting for 0.04347291662575192 secs before reviving daemons 2026-02-18T06:38:13.068 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:38:13.067+0000 7efdd695c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 29976) UID: 1000 2026-02-18T06:38:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 95183872 bytes 2026-02-18T06:38:13.112 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-18T06:38:13.112 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:38:13.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 121102336 bytes 2026-02-18T06:38:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 200929280 bytes 2026-02-18T06:38:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 235014144 bytes 2026-02-18T06:38:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 265382912 bytes 2026-02-18T06:38:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 273581056 bytes 2026-02-18T06:38:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 324206592 bytes 2026-02-18T06:38:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 331406336 bytes 2026-02-18T06:38:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 363979776 bytes 2026-02-18T06:38:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 410443776 bytes 2026-02-18T06:38:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 447579136 bytes 2026-02-18T06:38:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 514706432 bytes 2026-02-18T06:38:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 592657408 bytes 2026-02-18T06:38:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 623717376 bytes 2026-02-18T06:38:13.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 670717952 bytes 2026-02-18T06:38:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 748800000 bytes 2026-02-18T06:38:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 749444096 bytes 2026-02-18T06:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 783585280 bytes 2026-02-18T06:38:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 804590592 bytes 2026-02-18T06:38:13.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 868103168 bytes 2026-02-18T06:38:13.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 885212160 bytes 2026-02-18T06:38:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 900554752 bytes 2026-02-18T06:38:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 948793344 bytes 2026-02-18T06:38:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1014977536 bytes 2026-02-18T06:38:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1031322624 bytes 2026-02-18T06:38:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1055866880 bytes 2026-02-18T06:38:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1091634176 bytes 2026-02-18T06:38:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1155615744 bytes 2026-02-18T06:38:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1197134848 bytes 2026-02-18T06:38:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1264986112 bytes 2026-02-18T06:38:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1306171392 bytes 2026-02-18T06:38:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1328861184 bytes 2026-02-18T06:38:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1409460224 bytes 2026-02-18T06:38:14.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1447908352 bytes 2026-02-18T06:38:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1474900992 bytes 2026-02-18T06:38:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1513999360 bytes 2026-02-18T06:38:14.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1604462592 bytes 2026-02-18T06:38:14.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1642127360 bytes 2026-02-18T06:38:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1693648896 bytes 2026-02-18T06:38:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1728525312 bytes 2026-02-18T06:38:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1779334144 bytes 2026-02-18T06:38:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1802660864 bytes 2026-02-18T06:38:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1818826752 bytes 2026-02-18T06:38:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1858765824 bytes 2026-02-18T06:38:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1894960128 bytes 2026-02-18T06:38:15.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1922349056 bytes 2026-02-18T06:38:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2002822144 bytes 2026-02-18T06:38:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2044821504 bytes 2026-02-18T06:38:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2152856576 bytes 2026-02-18T06:38:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2233499648 bytes 2026-02-18T06:38:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2295949312 bytes 2026-02-18T06:38:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2342011904 bytes 2026-02-18T06:38:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2352673792 bytes 2026-02-18T06:38:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2353278976 bytes 2026-02-18T06:38:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2415372288 bytes 2026-02-18T06:38:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2428240896 bytes 2026-02-18T06:38:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2481632256 bytes 2026-02-18T06:38:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2499366912 bytes 2026-02-18T06:38:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2542099456 bytes 2026-02-18T06:38:15.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2581671936 bytes 2026-02-18T06:38:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2628503552 bytes 2026-02-18T06:38:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2650629120 bytes 2026-02-18T06:38:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2652924928 bytes 2026-02-18T06:38:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2688316416 bytes 2026-02-18T06:38:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2709241856 bytes 2026-02-18T06:38:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2738111488 bytes 2026-02-18T06:38:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2792111104 bytes 2026-02-18T06:38:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2857202688 bytes 2026-02-18T06:38:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2896689152 bytes 2026-02-18T06:38:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2930061312 bytes 2026-02-18T06:38:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2984035328 bytes 2026-02-18T06:38:16.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3045044224 bytes 2026-02-18T06:38:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3087232000 bytes 2026-02-18T06:38:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3135662080 bytes 2026-02-18T06:38:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3174628352 bytes 2026-02-18T06:38:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3275681792 bytes 2026-02-18T06:38:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3322692608 bytes 2026-02-18T06:38:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3364234240 bytes 2026-02-18T06:38:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3379214336 bytes 2026-02-18T06:38:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3424144384 bytes 2026-02-18T06:38:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3491728384 bytes 2026-02-18T06:38:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3542633472 bytes 2026-02-18T06:38:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3554279424 bytes 2026-02-18T06:38:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3629064192 bytes 2026-02-18T06:38:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3670319104 bytes 2026-02-18T06:38:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3722799104 bytes 2026-02-18T06:38:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3743358976 bytes 2026-02-18T06:38:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3805405184 bytes 2026-02-18T06:38:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3860732928 bytes 2026-02-18T06:38:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3915476992 bytes 2026-02-18T06:38:17.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3968670720 bytes 2026-02-18T06:38:17.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4008274944 bytes 2026-02-18T06:38:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4039421952 bytes 2026-02-18T06:38:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4064309248 bytes 2026-02-18T06:38:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4071298048 bytes 2026-02-18T06:38:17.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4120787968 bytes 2026-02-18T06:38:17.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4166352896 bytes 2026-02-18T06:38:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4182262784 bytes 2026-02-18T06:38:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4291021824 bytes 2026-02-18T06:38:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4327660544 bytes 2026-02-18T06:38:17.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4331406336 bytes 2026-02-18T06:38:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4382095360 bytes 2026-02-18T06:38:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4547710976 bytes 2026-02-18T06:38:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4618656768 bytes 2026-02-18T06:38:17.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4642685952 bytes 2026-02-18T06:38:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4772057088 bytes 2026-02-18T06:38:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4788120576 bytes 2026-02-18T06:38:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4816865280 bytes 2026-02-18T06:38:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4843975680 bytes 2026-02-18T06:38:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4937102336 bytes 2026-02-18T06:38:18.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5009489920 bytes 2026-02-18T06:38:18.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5112680448 bytes 2026-02-18T06:38:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5124208640 bytes 2026-02-18T06:38:18.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5160258560 bytes 2026-02-18T06:38:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5231619072 bytes 2026-02-18T06:38:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5293347840 bytes 2026-02-18T06:38:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5302018048 bytes 2026-02-18T06:38:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5376408576 bytes 2026-02-18T06:38:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5413775360 bytes 2026-02-18T06:38:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5428215808 bytes 2026-02-18T06:38:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5467993088 bytes 2026-02-18T06:38:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5505112064 bytes 2026-02-18T06:38:18.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5588526080 bytes 2026-02-18T06:38:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5621947392 bytes 2026-02-18T06:38:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5636680704 bytes 2026-02-18T06:38:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5689274368 bytes 2026-02-18T06:38:18.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5698113536 bytes 2026-02-18T06:38:18.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5753320448 bytes 2026-02-18T06:38:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5820355584 bytes 2026-02-18T06:38:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5887227904 bytes 2026-02-18T06:38:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5902646272 bytes 2026-02-18T06:38:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5916359680 bytes 2026-02-18T06:38:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5916955648 bytes 2026-02-18T06:38:19.213 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-18T06:38:19.213 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-18T06:38:19.214 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-18T06:38:19.217 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-18T06:38:19.217 INFO:tasks.rbd_mirror_thrash:waiting for 79.59669367517458 secs before thrashing 2026-02-18T06:38:19.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5977635840 bytes 2026-02-18T06:38:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5996553216 bytes 2026-02-18T06:38:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6036802560 bytes 2026-02-18T06:38:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6111579136 bytes 2026-02-18T06:38:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6138737664 bytes 2026-02-18T06:38:19.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6154520576 bytes 2026-02-18T06:38:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6193292288 bytes 2026-02-18T06:38:19.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6260254720 bytes 2026-02-18T06:38:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6305387520 bytes 2026-02-18T06:38:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6376200192 bytes 2026-02-18T06:38:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6423084032 bytes 2026-02-18T06:38:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6435244032 bytes 2026-02-18T06:38:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6486649856 bytes 2026-02-18T06:38:19.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6489860096 bytes 2026-02-18T06:38:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6521884672 bytes 2026-02-18T06:38:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6551786496 bytes 2026-02-18T06:38:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6579480576 bytes 2026-02-18T06:38:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6606997504 bytes 2026-02-18T06:38:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6615901184 bytes 2026-02-18T06:38:20.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6651784192 bytes 2026-02-18T06:38:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6686990336 bytes 2026-02-18T06:38:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6707228672 bytes 2026-02-18T06:38:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6753510400 bytes 2026-02-18T06:38:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6800790528 bytes 2026-02-18T06:38:20.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6864045056 bytes 2026-02-18T06:38:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6898637824 bytes 2026-02-18T06:38:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6935757824 bytes 2026-02-18T06:38:20.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6968006656 bytes 2026-02-18T06:38:20.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6996378624 bytes 2026-02-18T06:38:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7008684032 bytes 2026-02-18T06:38:20.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7033188352 bytes 2026-02-18T06:38:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7068318720 bytes 2026-02-18T06:38:20.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7097440256 bytes 2026-02-18T06:38:20.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7177926656 bytes 2026-02-18T06:38:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7259149312 bytes 2026-02-18T06:38:20.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7272145920 bytes 2026-02-18T06:38:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7297068032 bytes 2026-02-18T06:38:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7339974656 bytes 2026-02-18T06:38:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7352162304 bytes 2026-02-18T06:38:20.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7353577472 bytes 2026-02-18T06:38:20.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7368309760 bytes 2026-02-18T06:38:20.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7405775872 bytes 2026-02-18T06:38:20.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7438147584 bytes 2026-02-18T06:38:21.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7501364224 bytes 2026-02-18T06:38:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7523359744 bytes 2026-02-18T06:38:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7558214656 bytes 2026-02-18T06:38:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7591624704 bytes 2026-02-18T06:38:21.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7606134784 bytes 2026-02-18T06:38:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7661618176 bytes 2026-02-18T06:38:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7725959168 bytes 2026-02-18T06:38:21.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7726593024 bytes 2026-02-18T06:38:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7786712064 bytes 2026-02-18T06:38:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7814145024 bytes 2026-02-18T06:38:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7828175872 bytes 2026-02-18T06:38:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7875904512 bytes 2026-02-18T06:38:21.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7929993216 bytes 2026-02-18T06:38:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7942508544 bytes 2026-02-18T06:38:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7963856896 bytes 2026-02-18T06:38:21.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8029755392 bytes 2026-02-18T06:38:21.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8071438336 bytes 2026-02-18T06:38:21.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8107523072 bytes 2026-02-18T06:38:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8125989888 bytes 2026-02-18T06:38:21.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8161427456 bytes 2026-02-18T06:38:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8252920832 bytes 2026-02-18T06:38:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8305565696 bytes 2026-02-18T06:38:22.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8366994432 bytes 2026-02-18T06:38:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8385520640 bytes 2026-02-18T06:38:22.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8433581056 bytes 2026-02-18T06:38:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8476063744 bytes 2026-02-18T06:38:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8562407424 bytes 2026-02-18T06:38:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8621726720 bytes 2026-02-18T06:38:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8644325376 bytes 2026-02-18T06:38:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8652297216 bytes 2026-02-18T06:38:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8701714432 bytes 2026-02-18T06:38:22.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8761333760 bytes 2026-02-18T06:38:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8801161216 bytes 2026-02-18T06:38:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8836921344 bytes 2026-02-18T06:38:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8864154624 bytes 2026-02-18T06:38:22.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8915298304 bytes 2026-02-18T06:38:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 8980803584 bytes 2026-02-18T06:38:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 9050566656 bytes 2026-02-18T06:38:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 9086149632 bytes 2026-02-18T06:38:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 9106121728 bytes 2026-02-18T06:38:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 9215774720 bytes 2026-02-18T06:38:22.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 9239669760 bytes 2026-02-18T06:38:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 9240305664 bytes 2026-02-18T06:38:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 9324418048 bytes 2026-02-18T06:38:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 9355223040 bytes 2026-02-18T06:38:23.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 9382945792 bytes 2026-02-18T06:38:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 9426184192 bytes 2026-02-18T06:38:23.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 9440948224 bytes 2026-02-18T06:38:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 9539443712 bytes 2026-02-18T06:38:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 9571423232 bytes 2026-02-18T06:38:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 9607694336 bytes 2026-02-18T06:38:23.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 9690802176 bytes 2026-02-18T06:38:23.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23000 IOs, 9709196288 bytes 2026-02-18T06:38:23.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23100 IOs, 9765735424 bytes 2026-02-18T06:38:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23200 IOs, 9786882048 bytes 2026-02-18T06:38:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23300 IOs, 9801951232 bytes 2026-02-18T06:38:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23400 IOs, 9852348416 bytes 2026-02-18T06:38:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23500 IOs, 9852949504 bytes 2026-02-18T06:38:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23600 IOs, 9867703296 bytes 2026-02-18T06:38:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23700 IOs, 9926097920 bytes 2026-02-18T06:38:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23800 IOs, 9973708800 bytes 2026-02-18T06:38:23.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23900 IOs, 10029481984 bytes 2026-02-18T06:38:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24000 IOs, 10052323328 bytes 2026-02-18T06:38:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24100 IOs, 10077217792 bytes 2026-02-18T06:38:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24200 IOs, 10142028800 bytes 2026-02-18T06:38:24.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24300 IOs, 10162202624 bytes 2026-02-18T06:38:24.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24400 IOs, 10248219648 bytes 2026-02-18T06:38:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24500 IOs, 10267301888 bytes 2026-02-18T06:38:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24600 IOs, 10307222528 bytes 2026-02-18T06:38:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24700 IOs, 10360895488 bytes 2026-02-18T06:38:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24800 IOs, 10464220160 bytes 2026-02-18T06:38:24.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24900 IOs, 10489221120 bytes 2026-02-18T06:38:24.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25000 IOs, 10544397312 bytes 2026-02-18T06:38:24.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25100 IOs, 10608685056 bytes 2026-02-18T06:38:24.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25200 IOs, 10639498240 bytes 2026-02-18T06:38:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25300 IOs, 10707603456 bytes 2026-02-18T06:38:24.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25400 IOs, 10782159872 bytes 2026-02-18T06:38:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25500 IOs, 10839099392 bytes 2026-02-18T06:38:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25600 IOs, 10925523968 bytes 2026-02-18T06:38:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25700 IOs, 10988627968 bytes 2026-02-18T06:38:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25800 IOs, 11047052288 bytes 2026-02-18T06:38:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:25900 IOs, 11093192704 bytes 2026-02-18T06:38:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26000 IOs, 11103063040 bytes 2026-02-18T06:38:25.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26100 IOs, 11122920448 bytes 2026-02-18T06:38:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26200 IOs, 11123524608 bytes 2026-02-18T06:38:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26300 IOs, 11149310976 bytes 2026-02-18T06:38:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26400 IOs, 11238188032 bytes 2026-02-18T06:38:25.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26500 IOs, 11331933184 bytes 2026-02-18T06:38:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26600 IOs, 11413265408 bytes 2026-02-18T06:38:25.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26700 IOs, 11502206976 bytes 2026-02-18T06:38:25.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26800 IOs, 11537251328 bytes 2026-02-18T06:38:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:26900 IOs, 11537870848 bytes 2026-02-18T06:38:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27000 IOs, 11557809152 bytes 2026-02-18T06:38:25.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27100 IOs, 11589972992 bytes 2026-02-18T06:38:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27200 IOs, 11590561792 bytes 2026-02-18T06:38:25.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27300 IOs, 11619738624 bytes 2026-02-18T06:38:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27400 IOs, 11627034624 bytes 2026-02-18T06:38:25.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27500 IOs, 11699347456 bytes 2026-02-18T06:38:25.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27600 IOs, 11722847232 bytes 2026-02-18T06:38:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27700 IOs, 11747792896 bytes 2026-02-18T06:38:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27800 IOs, 11767387136 bytes 2026-02-18T06:38:25.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:27900 IOs, 11783163904 bytes 2026-02-18T06:38:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:28000 IOs, 11837892608 bytes 2026-02-18T06:38:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:28100 IOs, 11871840256 bytes 2026-02-18T06:38:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:28200 IOs, 11906563072 bytes 2026-02-18T06:38:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:38:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:38:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:38:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:38:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:38:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:38:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:38:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:38:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:38:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:38:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:38:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:38:26.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:38:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":267909632.0,"last_snapshot_bytes":535819264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396637,"remote_snapshot_timestamp":1771396637,"replay_state":"idle"} 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:38:07 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:38:08' 2026-02-18T06:38:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:38:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:38:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:38:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":267909632.0,"last_snapshot_bytes":535819264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396637,"remote_snapshot_timestamp":1771396637,"replay_state":"idle"} 2026-02-18T06:38:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:38:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:38:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:38:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:38:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:38:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:38:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:38:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:38:07' 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:38:07' 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:38:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:38:07' 2026-02-18T06:38:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:38:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:38:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:38:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK43814384mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":267909632.0,"last_snapshot_bytes":535819264,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396637,"remote_snapshot_timestamp":1771396637,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:38:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:38:08' 2026-02-18T06:38:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:38:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:38:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:38:07' 2026-02-18T06:38:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:38:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:38:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:38:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap3 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap3 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3 2026-02-18T06:38:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:38:55.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7272449649828542) > weight (0.25) 2026-02-18T06:38:55.757 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3566146581624834) > weight (0.25) 2026-02-18T06:38:55.757 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9716263401407026) > weight (0.25) 2026-02-18T06:38:55.757 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.43565943490316417) > weight (0.25) 2026-02-18T06:38:55.757 INFO:tasks.rbd_mirror_thrash:waiting for 89.98822702813318 secs before thrashing 2026-02-18T06:38:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:38:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:38:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:38:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:38:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:38:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:38:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:38:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:38:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:38:56.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:38:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:38:56.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:38:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:38:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:38:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:38:57.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:38:57.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:38:57.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:38:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:38:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:38:57.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:38:57.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:38:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:38:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:38:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:38:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:38:58.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 11 2026-02-18T06:38:58.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:38:58.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:38:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:38:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:38:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:38:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:58.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:38:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=11 2026-02-18T06:38:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:38:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:38:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:38:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:38:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 11 = '' 2026-02-18T06:38:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:38:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:38:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:38:59.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:38:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:38:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:38:59.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:38:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:38:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:38:59.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 11 = '' 2026-02-18T06:38:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:38:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:00.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:39:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:39:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:39:00.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:39:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:39:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 11 = '' 2026-02-18T06:39:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:39:00.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:39:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=11 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 11 = 11 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:39:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:39:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:39:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-18T06:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 6 2026-02-18T06:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:38:10 2026 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:39:03.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:39:03.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:39:03.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap2-clean ']' 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2-clean 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:39:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:03.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:03.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:04.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:04.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap3 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:04.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:05.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap3-clean 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3-clean 2026-02-18T06:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:05.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:39:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T06:39:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:39:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:39:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 40312832 bytes 2026-02-18T06:39:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 88230912 bytes 2026-02-18T06:39:06.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 201915392 bytes 2026-02-18T06:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 271226880 bytes 2026-02-18T06:39:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 289732608 bytes 2026-02-18T06:39:06.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 319283200 bytes 2026-02-18T06:39:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 407849984 bytes 2026-02-18T06:39:06.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 413245440 bytes 2026-02-18T06:39:06.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 418025472 bytes 2026-02-18T06:39:06.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 458946560 bytes 2026-02-18T06:39:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 493404160 bytes 2026-02-18T06:39:06.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 589733888 bytes 2026-02-18T06:39:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 673370112 bytes 2026-02-18T06:39:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 716286976 bytes 2026-02-18T06:39:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 744294400 bytes 2026-02-18T06:39:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 773337088 bytes 2026-02-18T06:39:06.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 798509056 bytes 2026-02-18T06:39:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 898875392 bytes 2026-02-18T06:39:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 996146176 bytes 2026-02-18T06:39:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1026187264 bytes 2026-02-18T06:39:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1063129088 bytes 2026-02-18T06:39:07.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1141278720 bytes 2026-02-18T06:39:07.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1186705408 bytes 2026-02-18T06:39:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1243408384 bytes 2026-02-18T06:39:07.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1301572608 bytes 2026-02-18T06:39:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1380109312 bytes 2026-02-18T06:39:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1409512448 bytes 2026-02-18T06:39:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1521007616 bytes 2026-02-18T06:39:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1531682816 bytes 2026-02-18T06:39:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1589026816 bytes 2026-02-18T06:39:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1623514112 bytes 2026-02-18T06:39:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1661915136 bytes 2026-02-18T06:39:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1712654336 bytes 2026-02-18T06:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1785612288 bytes 2026-02-18T06:39:07.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1825235968 bytes 2026-02-18T06:39:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1855381504 bytes 2026-02-18T06:39:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1938196480 bytes 2026-02-18T06:39:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1967916032 bytes 2026-02-18T06:39:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 2022963200 bytes 2026-02-18T06:39:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2047506432 bytes 2026-02-18T06:39:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2049290240 bytes 2026-02-18T06:39:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2065633280 bytes 2026-02-18T06:39:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2108328960 bytes 2026-02-18T06:39:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2126242816 bytes 2026-02-18T06:39:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2161275904 bytes 2026-02-18T06:39:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2186329088 bytes 2026-02-18T06:39:08.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2237089792 bytes 2026-02-18T06:39:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2348640256 bytes 2026-02-18T06:39:08.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2416030720 bytes 2026-02-18T06:39:08.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2455044096 bytes 2026-02-18T06:39:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2474108928 bytes 2026-02-18T06:39:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2514361344 bytes 2026-02-18T06:39:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2541869056 bytes 2026-02-18T06:39:08.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2619108352 bytes 2026-02-18T06:39:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2632468480 bytes 2026-02-18T06:39:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2646713344 bytes 2026-02-18T06:39:09.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2728799232 bytes 2026-02-18T06:39:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2747683840 bytes 2026-02-18T06:39:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2776403968 bytes 2026-02-18T06:39:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2801418240 bytes 2026-02-18T06:39:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2844267520 bytes 2026-02-18T06:39:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2849503232 bytes 2026-02-18T06:39:09.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2884835328 bytes 2026-02-18T06:39:09.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2909315072 bytes 2026-02-18T06:39:09.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2937921536 bytes 2026-02-18T06:39:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3020725248 bytes 2026-02-18T06:39:09.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3075755008 bytes 2026-02-18T06:39:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3087801344 bytes 2026-02-18T06:39:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3146072064 bytes 2026-02-18T06:39:09.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3252582400 bytes 2026-02-18T06:39:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3327337472 bytes 2026-02-18T06:39:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3399287808 bytes 2026-02-18T06:39:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3431025664 bytes 2026-02-18T06:39:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3466363904 bytes 2026-02-18T06:39:09.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3486138368 bytes 2026-02-18T06:39:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3508153344 bytes 2026-02-18T06:39:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3536093184 bytes 2026-02-18T06:39:10.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3580419072 bytes 2026-02-18T06:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3581008896 bytes 2026-02-18T06:39:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3637516288 bytes 2026-02-18T06:39:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3700303872 bytes 2026-02-18T06:39:10.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3722149888 bytes 2026-02-18T06:39:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3763139584 bytes 2026-02-18T06:39:10.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3796814848 bytes 2026-02-18T06:39:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3835757568 bytes 2026-02-18T06:39:10.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3905997824 bytes 2026-02-18T06:39:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3952131072 bytes 2026-02-18T06:39:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4032143360 bytes 2026-02-18T06:39:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4080884736 bytes 2026-02-18T06:39:10.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4114426880 bytes 2026-02-18T06:39:10.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4133623808 bytes 2026-02-18T06:39:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4158881792 bytes 2026-02-18T06:39:10.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4168186880 bytes 2026-02-18T06:39:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4227210240 bytes 2026-02-18T06:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4228047872 bytes 2026-02-18T06:39:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4260270080 bytes 2026-02-18T06:39:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4262536192 bytes 2026-02-18T06:39:10.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4300613632 bytes 2026-02-18T06:39:11.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4474997760 bytes 2026-02-18T06:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4503120896 bytes 2026-02-18T06:39:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4535436288 bytes 2026-02-18T06:39:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4588964864 bytes 2026-02-18T06:39:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4597221376 bytes 2026-02-18T06:39:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4627379200 bytes 2026-02-18T06:39:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4683333632 bytes 2026-02-18T06:39:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4752667648 bytes 2026-02-18T06:39:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4826078208 bytes 2026-02-18T06:39:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4909191168 bytes 2026-02-18T06:39:11.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4994666496 bytes 2026-02-18T06:39:11.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5029152768 bytes 2026-02-18T06:39:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5080444928 bytes 2026-02-18T06:39:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5141861376 bytes 2026-02-18T06:39:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5204377600 bytes 2026-02-18T06:39:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5330165760 bytes 2026-02-18T06:39:11.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5350056960 bytes 2026-02-18T06:39:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5364487168 bytes 2026-02-18T06:39:12.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5391216640 bytes 2026-02-18T06:39:12.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5457045504 bytes 2026-02-18T06:39:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5477167104 bytes 2026-02-18T06:39:12.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5523675136 bytes 2026-02-18T06:39:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5547415552 bytes 2026-02-18T06:39:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5580948480 bytes 2026-02-18T06:39:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5594296320 bytes 2026-02-18T06:39:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5660445696 bytes 2026-02-18T06:39:12.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5721096192 bytes 2026-02-18T06:39:12.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5762454528 bytes 2026-02-18T06:39:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5775939584 bytes 2026-02-18T06:39:12.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5825146880 bytes 2026-02-18T06:39:12.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5876337664 bytes 2026-02-18T06:39:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5882895360 bytes 2026-02-18T06:39:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5915633664 bytes 2026-02-18T06:39:12.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5951185920 bytes 2026-02-18T06:39:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6059622400 bytes 2026-02-18T06:39:12.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6060232704 bytes 2026-02-18T06:39:12.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6084594688 bytes 2026-02-18T06:39:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6153464832 bytes 2026-02-18T06:39:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6252875776 bytes 2026-02-18T06:39:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6318812160 bytes 2026-02-18T06:39:13.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6368106496 bytes 2026-02-18T06:39:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6407922688 bytes 2026-02-18T06:39:13.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6438325248 bytes 2026-02-18T06:39:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6506300416 bytes 2026-02-18T06:39:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6524958720 bytes 2026-02-18T06:39:13.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6575195136 bytes 2026-02-18T06:39:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6639720448 bytes 2026-02-18T06:39:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6640334848 bytes 2026-02-18T06:39:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6707293184 bytes 2026-02-18T06:39:13.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6816972800 bytes 2026-02-18T06:39:13.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6844104704 bytes 2026-02-18T06:39:13.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6871786496 bytes 2026-02-18T06:39:13.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6899150848 bytes 2026-02-18T06:39:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6913837056 bytes 2026-02-18T06:39:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6914978816 bytes 2026-02-18T06:39:13.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6951436288 bytes 2026-02-18T06:39:13.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 7041750016 bytes 2026-02-18T06:39:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 7136132096 bytes 2026-02-18T06:39:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7176625152 bytes 2026-02-18T06:39:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7228633088 bytes 2026-02-18T06:39:14.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7282079744 bytes 2026-02-18T06:39:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7321184256 bytes 2026-02-18T06:39:14.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7349738496 bytes 2026-02-18T06:39:14.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7374691328 bytes 2026-02-18T06:39:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7431906304 bytes 2026-02-18T06:39:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7453939712 bytes 2026-02-18T06:39:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7501914112 bytes 2026-02-18T06:39:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7569240064 bytes 2026-02-18T06:39:14.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7626736640 bytes 2026-02-18T06:39:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7656520704 bytes 2026-02-18T06:39:14.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7727808512 bytes 2026-02-18T06:39:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7755345920 bytes 2026-02-18T06:39:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7755960320 bytes 2026-02-18T06:39:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7819629568 bytes 2026-02-18T06:39:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7843241984 bytes 2026-02-18T06:39:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7885007872 bytes 2026-02-18T06:39:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:39:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:39:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:39:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:39:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:39:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:39:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:39:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"} 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:39:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:39:07 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:39:08' 2026-02-18T06:39:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:39:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:39:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:39:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"} 2026-02-18T06:39:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:39:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:39:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:39:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:39:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:39:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:39:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:39:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:39:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:39:07' 2026-02-18T06:39:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:39:07' 2026-02-18T06:39:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:39:07' 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:39:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:39:16.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK43814384mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:39:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:39:08' 2026-02-18T06:39:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:39:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:39:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:39:07' 2026-02-18T06:39:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:39:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:39:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:39:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:39:16.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap4 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap4 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4 2026-02-18T06:39:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:39:38.815 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8562983954780867) > weight (0.25) 2026-02-18T06:39:38.815 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.422209190367936) > weight (0.25) 2026-02-18T06:39:38.815 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7355372391311906) > weight (0.25) 2026-02-18T06:39:38.816 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8576738258661032) > weight (0.25) 2026-02-18T06:39:38.816 INFO:tasks.rbd_mirror_thrash:waiting for 101.14985449709715 secs before thrashing 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:39:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:39:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:39:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:39:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:39:45.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:39:45.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:39:45.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:39:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:39:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:39:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:39:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:39:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:39:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:39:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:39:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:39:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:39:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:39:45.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:39:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:39:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:39:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:39:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:39:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:39:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:39:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:39:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:39:45.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:39:46.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:39:46.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:39:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:39:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:39:46.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:39:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:39:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:39:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 14 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:39:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:39:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:39:47.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:39:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:39:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:39:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:39:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:39:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=14 2026-02-18T06:39:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:39:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:39:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:39:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:39:48.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:39:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:39:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:39:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:39:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 14 = '' 2026-02-18T06:39:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:39:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:39:48.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:48.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:39:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:39:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:39:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:39:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:39:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 14 = '' 2026-02-18T06:39:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:39:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:39:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:39:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:39:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:39:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:39:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:39:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 14 = '' 2026-02-18T06:39:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:39:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:39:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:51.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:39:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:39:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:39:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:39:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=14 2026-02-18T06:39:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 14 = 14 2026-02-18T06:39:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:39:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:39:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-18T06:39:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 8 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:39:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:38:10 2026 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:39:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:39:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap3-clean ']' 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3-clean 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:52.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:52.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:53.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap4 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4 2026-02-18T06:39:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:39:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:39:53.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:54.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:39:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:54.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:39:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap4-clean 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4-clean 2026-02-18T06:39:54.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:39:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:39:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:39:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:39:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 53214208 bytes 2026-02-18T06:39:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 68972544 bytes 2026-02-18T06:39:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 97622016 bytes 2026-02-18T06:39:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 145006592 bytes 2026-02-18T06:39:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 146920448 bytes 2026-02-18T06:39:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 183562240 bytes 2026-02-18T06:39:55.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 242881536 bytes 2026-02-18T06:39:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 282304512 bytes 2026-02-18T06:39:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 376258560 bytes 2026-02-18T06:39:55.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 442891264 bytes 2026-02-18T06:39:55.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 455129088 bytes 2026-02-18T06:39:55.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 499970048 bytes 2026-02-18T06:39:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 521096192 bytes 2026-02-18T06:39:55.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 557373440 bytes 2026-02-18T06:39:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 558012416 bytes 2026-02-18T06:39:55.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 606146560 bytes 2026-02-18T06:39:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 612454400 bytes 2026-02-18T06:39:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 635345920 bytes 2026-02-18T06:39:55.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 676559872 bytes 2026-02-18T06:39:55.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 722209792 bytes 2026-02-18T06:39:55.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 784462848 bytes 2026-02-18T06:39:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 858137600 bytes 2026-02-18T06:39:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 936265728 bytes 2026-02-18T06:39:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 979748864 bytes 2026-02-18T06:39:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 988814336 bytes 2026-02-18T06:39:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1063636992 bytes 2026-02-18T06:39:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1086495744 bytes 2026-02-18T06:39:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1116116992 bytes 2026-02-18T06:39:56.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1130072064 bytes 2026-02-18T06:39:56.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1155500032 bytes 2026-02-18T06:39:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1197287424 bytes 2026-02-18T06:39:56.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1239163904 bytes 2026-02-18T06:39:56.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1274434560 bytes 2026-02-18T06:39:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1287522304 bytes 2026-02-18T06:39:56.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1303573504 bytes 2026-02-18T06:39:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1351230464 bytes 2026-02-18T06:39:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1415134208 bytes 2026-02-18T06:39:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1415747584 bytes 2026-02-18T06:39:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1479105536 bytes 2026-02-18T06:39:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1496686592 bytes 2026-02-18T06:39:57.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1545472000 bytes 2026-02-18T06:39:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1579592704 bytes 2026-02-18T06:39:57.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1621471232 bytes 2026-02-18T06:39:57.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1622780928 bytes 2026-02-18T06:39:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1664540672 bytes 2026-02-18T06:39:57.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1685700608 bytes 2026-02-18T06:39:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1728997376 bytes 2026-02-18T06:39:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1792451584 bytes 2026-02-18T06:39:57.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1856267264 bytes 2026-02-18T06:39:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1883744256 bytes 2026-02-18T06:39:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1884868608 bytes 2026-02-18T06:39:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1916191744 bytes 2026-02-18T06:39:57.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1955484672 bytes 2026-02-18T06:39:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1983908864 bytes 2026-02-18T06:39:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2041036800 bytes 2026-02-18T06:39:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2082832384 bytes 2026-02-18T06:39:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2095059968 bytes 2026-02-18T06:39:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2143304704 bytes 2026-02-18T06:39:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2166142976 bytes 2026-02-18T06:39:58.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2225282048 bytes 2026-02-18T06:39:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2232411136 bytes 2026-02-18T06:39:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2273545216 bytes 2026-02-18T06:39:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2290360320 bytes 2026-02-18T06:39:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2315211776 bytes 2026-02-18T06:39:58.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2360833024 bytes 2026-02-18T06:39:58.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2380045312 bytes 2026-02-18T06:39:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2421492736 bytes 2026-02-18T06:39:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2444095488 bytes 2026-02-18T06:39:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2518733824 bytes 2026-02-18T06:39:58.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2531588096 bytes 2026-02-18T06:39:58.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2580116480 bytes 2026-02-18T06:39:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2607187968 bytes 2026-02-18T06:39:58.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2656334848 bytes 2026-02-18T06:39:58.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2720924672 bytes 2026-02-18T06:39:58.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2770087936 bytes 2026-02-18T06:39:58.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2781423616 bytes 2026-02-18T06:39:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2784116736 bytes 2026-02-18T06:39:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2860394496 bytes 2026-02-18T06:39:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2877032448 bytes 2026-02-18T06:39:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2925248512 bytes 2026-02-18T06:39:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 2958063616 bytes 2026-02-18T06:39:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 2972459008 bytes 2026-02-18T06:39:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3010203648 bytes 2026-02-18T06:39:59.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3063585792 bytes 2026-02-18T06:39:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3065969664 bytes 2026-02-18T06:39:59.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3092855808 bytes 2026-02-18T06:39:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3120553984 bytes 2026-02-18T06:39:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3159458816 bytes 2026-02-18T06:39:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3193106432 bytes 2026-02-18T06:39:59.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3253215232 bytes 2026-02-18T06:39:59.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3311343616 bytes 2026-02-18T06:39:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3395493888 bytes 2026-02-18T06:39:59.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3456637952 bytes 2026-02-18T06:39:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3534147584 bytes 2026-02-18T06:39:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3590770688 bytes 2026-02-18T06:39:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3646884864 bytes 2026-02-18T06:39:59.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3663762432 bytes 2026-02-18T06:39:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:39:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:39:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:39:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:39:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:40:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:40:00.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:40:00.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:40:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"} 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:39:37 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:40:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:39:38' 2026-02-18T06:40:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:40:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:40:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:40:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"} 2026-02-18T06:40:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:40:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:40:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:40:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:40:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:40:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:40:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:40:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:40:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:39:37' 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:39:37' 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:39:37' 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:40:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:40:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK43814384mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":17893580.8,"bytes_per_snapshot":535817216.0,"last_snapshot_bytes":536807424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396737,"remote_snapshot_timestamp":1771396737,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:39:37cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:39:38' 2026-02-18T06:40:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:40:01.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:40:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:39:37' 2026-02-18T06:40:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:40:01.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:40:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:40:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:40:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:40:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap5 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap5 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5 2026-02-18T06:40:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:40:25.746 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T06:40:25.746 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T06:40:25.746 INFO:tasks.rbd_mirror_thrash:waiting for 3.26192609060469 secs before reviving daemons 2026-02-18T06:40:25.747 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T06:40:25.746+0000 7fda0cb4c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 30023) UID: 1000 2026-02-18T06:40:27.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:40:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:40:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:40:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:40:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:40:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:40:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:40:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:40:28.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:40:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:40:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:40:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:40:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:40:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:40:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:40:29.009 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T06:40:29.009 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:40:29.010 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T06:40:29.010 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:40:29.010 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:40:29.012 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:40:29.013 INFO:tasks.rbd_mirror_thrash:waiting for 93.60595674717948 secs before thrashing 2026-02-18T06:40:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:40:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:40:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:40:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:40:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:40:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:40:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:40:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:40:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:40:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:40:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:40:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:40:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:40:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:40:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:40:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:40:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:40:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:40:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:40:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:40:30.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 17 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:40:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:40:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:40:30.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:40:30.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:40:30.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:40:30.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:40:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:40:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=17 2026-02-18T06:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:40:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:40:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:40:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:40:31.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:40:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:40:31.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:40:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:40:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:40:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:40:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 17 = '' 2026-02-18T06:40:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:40:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:40:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:40:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:40:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:40:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:40:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:40:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:40:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:40:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 17 = '' 2026-02-18T06:40:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:40:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:40:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:40:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:40:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:40:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:40:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:40:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:40:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:40:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:40:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 17 = '' 2026-02-18T06:40:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:40:32.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:40:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:40:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:40:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:40:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=17 2026-02-18T06:40:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 17 = 17 2026-02-18T06:40:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:40:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:40:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 10 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:40:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:39:52 2026 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:40:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap4-clean ']' 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4-clean 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:40:35.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:40:35.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:40:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap5 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:40:36.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:40:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:40:37.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:40:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap5-clean 2026-02-18T06:40:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-18T06:40:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5-clean 2026-02-18T06:40:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:40:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:40:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:40:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:37.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:40:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:40:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:40:37.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:40:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 26009600 bytes 2026-02-18T06:40:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 80059392 bytes 2026-02-18T06:40:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 96070656 bytes 2026-02-18T06:40:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 128695296 bytes 2026-02-18T06:40:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 213482496 bytes 2026-02-18T06:40:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 230950912 bytes 2026-02-18T06:40:38.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 266268672 bytes 2026-02-18T06:40:38.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 311404544 bytes 2026-02-18T06:40:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 438522880 bytes 2026-02-18T06:40:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 466880512 bytes 2026-02-18T06:40:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 504153088 bytes 2026-02-18T06:40:38.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 560512000 bytes 2026-02-18T06:40:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 587849728 bytes 2026-02-18T06:40:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 629913600 bytes 2026-02-18T06:40:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 677827584 bytes 2026-02-18T06:40:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 702288896 bytes 2026-02-18T06:40:39.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 739963904 bytes 2026-02-18T06:40:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 771879936 bytes 2026-02-18T06:40:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 777899008 bytes 2026-02-18T06:40:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 814056448 bytes 2026-02-18T06:40:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 814661632 bytes 2026-02-18T06:40:39.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 876345344 bytes 2026-02-18T06:40:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1002766336 bytes 2026-02-18T06:40:39.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1076817920 bytes 2026-02-18T06:40:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1094787072 bytes 2026-02-18T06:40:39.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1128150016 bytes 2026-02-18T06:40:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1144295424 bytes 2026-02-18T06:40:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1169449984 bytes 2026-02-18T06:40:39.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1205785600 bytes 2026-02-18T06:40:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1246124032 bytes 2026-02-18T06:40:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1250390016 bytes 2026-02-18T06:40:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1256218624 bytes 2026-02-18T06:40:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1256818688 bytes 2026-02-18T06:40:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1347116032 bytes 2026-02-18T06:40:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1366504448 bytes 2026-02-18T06:40:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1465064448 bytes 2026-02-18T06:40:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1499805696 bytes 2026-02-18T06:40:40.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1537580032 bytes 2026-02-18T06:40:40.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1562114048 bytes 2026-02-18T06:40:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1631323136 bytes 2026-02-18T06:40:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1643608064 bytes 2026-02-18T06:40:40.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1706921984 bytes 2026-02-18T06:40:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1707534336 bytes 2026-02-18T06:40:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1718821888 bytes 2026-02-18T06:40:40.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1765645312 bytes 2026-02-18T06:40:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1824225280 bytes 2026-02-18T06:40:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1894860800 bytes 2026-02-18T06:40:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1911568384 bytes 2026-02-18T06:40:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2048771072 bytes 2026-02-18T06:40:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2084914176 bytes 2026-02-18T06:40:40.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2111764480 bytes 2026-02-18T06:40:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2163880960 bytes 2026-02-18T06:40:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2231483392 bytes 2026-02-18T06:40:41.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2274892800 bytes 2026-02-18T06:40:41.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2316199936 bytes 2026-02-18T06:40:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2327917568 bytes 2026-02-18T06:40:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2395670528 bytes 2026-02-18T06:40:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2422833152 bytes 2026-02-18T06:40:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2476450816 bytes 2026-02-18T06:40:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2494588928 bytes 2026-02-18T06:40:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2518749184 bytes 2026-02-18T06:40:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2622821376 bytes 2026-02-18T06:40:41.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2667703296 bytes 2026-02-18T06:40:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2752434176 bytes 2026-02-18T06:40:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2764190720 bytes 2026-02-18T06:40:41.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2798497792 bytes 2026-02-18T06:40:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2818312192 bytes 2026-02-18T06:40:41.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2841774080 bytes 2026-02-18T06:40:41.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2859456512 bytes 2026-02-18T06:40:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2897629184 bytes 2026-02-18T06:40:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2975613952 bytes 2026-02-18T06:40:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2992700416 bytes 2026-02-18T06:40:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3030398976 bytes 2026-02-18T06:40:42.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3095025664 bytes 2026-02-18T06:40:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3103930368 bytes 2026-02-18T06:40:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3133849600 bytes 2026-02-18T06:40:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3178824704 bytes 2026-02-18T06:40:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3179464704 bytes 2026-02-18T06:40:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3204320256 bytes 2026-02-18T06:40:42.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3204908032 bytes 2026-02-18T06:40:42.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3239522304 bytes 2026-02-18T06:40:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3315042304 bytes 2026-02-18T06:40:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3353546752 bytes 2026-02-18T06:40:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3392986112 bytes 2026-02-18T06:40:42.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3470924800 bytes 2026-02-18T06:40:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3567003648 bytes 2026-02-18T06:40:42.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3601485824 bytes 2026-02-18T06:40:42.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3619771392 bytes 2026-02-18T06:40:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3647592448 bytes 2026-02-18T06:40:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3695739904 bytes 2026-02-18T06:40:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3732051968 bytes 2026-02-18T06:40:42.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3806707712 bytes 2026-02-18T06:40:43.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3843659776 bytes 2026-02-18T06:40:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3923262464 bytes 2026-02-18T06:40:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3979904000 bytes 2026-02-18T06:40:43.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4064208896 bytes 2026-02-18T06:40:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4081438720 bytes 2026-02-18T06:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4159280128 bytes 2026-02-18T06:40:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4203357184 bytes 2026-02-18T06:40:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4287886336 bytes 2026-02-18T06:40:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4308085760 bytes 2026-02-18T06:40:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4349561856 bytes 2026-02-18T06:40:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4401584128 bytes 2026-02-18T06:40:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4419908608 bytes 2026-02-18T06:40:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4531372032 bytes 2026-02-18T06:40:43.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4594840576 bytes 2026-02-18T06:40:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4631448576 bytes 2026-02-18T06:40:43.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4665111552 bytes 2026-02-18T06:40:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4689162240 bytes 2026-02-18T06:40:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4708673536 bytes 2026-02-18T06:40:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4748442624 bytes 2026-02-18T06:40:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4822470656 bytes 2026-02-18T06:40:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4876929024 bytes 2026-02-18T06:40:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4877555712 bytes 2026-02-18T06:40:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4894524416 bytes 2026-02-18T06:40:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5011159040 bytes 2026-02-18T06:40:44.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5077123072 bytes 2026-02-18T06:40:44.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5102627840 bytes 2026-02-18T06:40:44.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5103268864 bytes 2026-02-18T06:40:44.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5158688768 bytes 2026-02-18T06:40:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5159265280 bytes 2026-02-18T06:40:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5236670464 bytes 2026-02-18T06:40:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5257179136 bytes 2026-02-18T06:40:44.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5310981120 bytes 2026-02-18T06:40:44.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5354566656 bytes 2026-02-18T06:40:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5384173568 bytes 2026-02-18T06:40:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5390523392 bytes 2026-02-18T06:40:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5405603840 bytes 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:40:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:40:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:40:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:40:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:40:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:40:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:40:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:40:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:40:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:40:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:40:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:40:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":4753544.53,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"} 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:40:37 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:40:38' 2026-02-18T06:40:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:40:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:40:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:40:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":4753544.53,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"} 2026-02-18T06:40:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:40:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:40:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:40:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:40:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:40:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:40:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:40:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:40:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:40:37' 2026-02-18T06:40:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:40:37' 2026-02-18T06:40:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:40:37' 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:40:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:40:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK43814384mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":4753544.53,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:40:37cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:40:38' 2026-02-18T06:40:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:40:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:40:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:40:37' 2026-02-18T06:40:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:40:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:40:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:40:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:40:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap6 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap6 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:40:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:40:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:40:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6 2026-02-18T06:40:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:41:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:41:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:41:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:41:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:41:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:41:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:41:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:41:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:41:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:41:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:41:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:41:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:41:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:41:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:41:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:41:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:41:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:41:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:41:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:41:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:41:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:41:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:41:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:41:14.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:41:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:41:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:41:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:41:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:41:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:41:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:41:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:41:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:41:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:41:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:41:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:41:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:41:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 20 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:41:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:41:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:41:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:41:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:41:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:41:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:41:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:41:16.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:41:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=20 2026-02-18T06:41:16.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:41:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:41:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:41:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:41:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:41:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:41:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:41:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:41:17.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 20 = '' 2026-02-18T06:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:41:17.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:41:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:41:17.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:41:17.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:41:17.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:41:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:41:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:41:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:41:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 20 = '' 2026-02-18T06:41:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:41:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:41:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:41:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:41:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:41:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:41:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:41:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:41:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:41:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 20 = '' 2026-02-18T06:41:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:41:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:41:19.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5513312028562711) > weight (0.25) 2026-02-18T06:41:19.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9806779958119761) > weight (0.25) 2026-02-18T06:41:19.966 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6625342729115796) > weight (0.25) 2026-02-18T06:41:19.967 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-18T06:41:19.967 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-18T06:41:19.967 INFO:tasks.rbd_mirror_thrash:waiting for 5.90430916463312 secs before reviving daemons 2026-02-18T06:41:19.968 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:41:19.967+0000 7f1a497a1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 33292) UID: 1000 2026-02-18T06:41:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:41:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:41:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:41:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=20 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 20 = 20 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:41:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:41:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-18T06:41:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:41:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:41:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:41:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 12 2026-02-18T06:41:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:39:52 2026 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:41:21.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:41:21.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap5-clean ']' 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5-clean 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:41:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:41:21.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:21.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:21.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:41:21.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:41:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:22.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:41:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap6 2026-02-18T06:41:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:41:22.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:41:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:41:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:23.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:41:23.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap6-clean 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6-clean 2026-02-18T06:41:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:41:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:41:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:41:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:41:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T06:41:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:41:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:41:24.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 33353728 bytes 2026-02-18T06:41:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 127601664 bytes 2026-02-18T06:41:24.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 128209920 bytes 2026-02-18T06:41:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 149565440 bytes 2026-02-18T06:41:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 183089152 bytes 2026-02-18T06:41:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 279909376 bytes 2026-02-18T06:41:24.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 328212480 bytes 2026-02-18T06:41:24.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 362652672 bytes 2026-02-18T06:41:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 370937856 bytes 2026-02-18T06:41:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 434315264 bytes 2026-02-18T06:41:24.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 476081152 bytes 2026-02-18T06:41:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 550004736 bytes 2026-02-18T06:41:24.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 618735616 bytes 2026-02-18T06:41:24.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 661520384 bytes 2026-02-18T06:41:24.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 692928512 bytes 2026-02-18T06:41:24.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 730760192 bytes 2026-02-18T06:41:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 767596544 bytes 2026-02-18T06:41:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 822395904 bytes 2026-02-18T06:41:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 870725632 bytes 2026-02-18T06:41:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 900824064 bytes 2026-02-18T06:41:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 964877312 bytes 2026-02-18T06:41:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 983365632 bytes 2026-02-18T06:41:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1014230016 bytes 2026-02-18T06:41:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1082146816 bytes 2026-02-18T06:41:25.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1107458048 bytes 2026-02-18T06:41:25.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1135677440 bytes 2026-02-18T06:41:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1150174208 bytes 2026-02-18T06:41:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1265432576 bytes 2026-02-18T06:41:25.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1277786112 bytes 2026-02-18T06:41:25.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1368161280 bytes 2026-02-18T06:41:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1383546880 bytes 2026-02-18T06:41:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1426092032 bytes 2026-02-18T06:41:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1488320512 bytes 2026-02-18T06:41:25.872 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-18T06:41:25.873 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:41:25.873 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-18T06:41:25.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-18T06:41:25.873 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-18T06:41:25.876 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-18T06:41:25.876 INFO:tasks.rbd_mirror_thrash:waiting for 79.617763845867 secs before thrashing 2026-02-18T06:41:25.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1551372288 bytes 2026-02-18T06:41:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1569133568 bytes 2026-02-18T06:41:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1626357760 bytes 2026-02-18T06:41:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1703996416 bytes 2026-02-18T06:41:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1725556736 bytes 2026-02-18T06:41:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1739284480 bytes 2026-02-18T06:41:26.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1780273152 bytes 2026-02-18T06:41:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1804094464 bytes 2026-02-18T06:41:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1929528320 bytes 2026-02-18T06:41:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1967609856 bytes 2026-02-18T06:41:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2042403840 bytes 2026-02-18T06:41:26.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2104455168 bytes 2026-02-18T06:41:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2155686912 bytes 2026-02-18T06:41:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2168345600 bytes 2026-02-18T06:41:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2236729344 bytes 2026-02-18T06:41:26.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2275581952 bytes 2026-02-18T06:41:26.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2289902592 bytes 2026-02-18T06:41:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2307716096 bytes 2026-02-18T06:41:27.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2309229568 bytes 2026-02-18T06:41:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2337869824 bytes 2026-02-18T06:41:27.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2361335808 bytes 2026-02-18T06:41:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2418940928 bytes 2026-02-18T06:41:27.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2461857792 bytes 2026-02-18T06:41:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2520043520 bytes 2026-02-18T06:41:27.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2551861248 bytes 2026-02-18T06:41:27.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2592608256 bytes 2026-02-18T06:41:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2645883904 bytes 2026-02-18T06:41:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2677499904 bytes 2026-02-18T06:41:27.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2678092800 bytes 2026-02-18T06:41:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2739312640 bytes 2026-02-18T06:41:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2795917312 bytes 2026-02-18T06:41:27.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2837209088 bytes 2026-02-18T06:41:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2863149056 bytes 2026-02-18T06:41:27.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2881933312 bytes 2026-02-18T06:41:27.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2922135552 bytes 2026-02-18T06:41:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2952854528 bytes 2026-02-18T06:41:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2967140352 bytes 2026-02-18T06:41:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2997155840 bytes 2026-02-18T06:41:28.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3032825856 bytes 2026-02-18T06:41:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3038504960 bytes 2026-02-18T06:41:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3082701824 bytes 2026-02-18T06:41:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3084261376 bytes 2026-02-18T06:41:28.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3134912512 bytes 2026-02-18T06:41:28.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3135528960 bytes 2026-02-18T06:41:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3181746176 bytes 2026-02-18T06:41:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3264482304 bytes 2026-02-18T06:41:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3266242560 bytes 2026-02-18T06:41:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3355714560 bytes 2026-02-18T06:41:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3387491328 bytes 2026-02-18T06:41:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3430041600 bytes 2026-02-18T06:41:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3455719424 bytes 2026-02-18T06:41:28.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3518350336 bytes 2026-02-18T06:41:28.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3542690816 bytes 2026-02-18T06:41:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3545302016 bytes 2026-02-18T06:41:28.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3598999552 bytes 2026-02-18T06:41:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3677602816 bytes 2026-02-18T06:41:28.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3678228480 bytes 2026-02-18T06:41:28.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3710990336 bytes 2026-02-18T06:41:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3748700160 bytes 2026-02-18T06:41:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3793508352 bytes 2026-02-18T06:41:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3825787904 bytes 2026-02-18T06:41:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3888188416 bytes 2026-02-18T06:41:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3972892672 bytes 2026-02-18T06:41:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3999079424 bytes 2026-02-18T06:41:29.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4029038592 bytes 2026-02-18T06:41:29.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4036755456 bytes 2026-02-18T06:41:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4072304640 bytes 2026-02-18T06:41:29.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4125169664 bytes 2026-02-18T06:41:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4164153344 bytes 2026-02-18T06:41:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4210394112 bytes 2026-02-18T06:41:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4212916224 bytes 2026-02-18T06:41:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4219076608 bytes 2026-02-18T06:41:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4280856576 bytes 2026-02-18T06:41:29.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4342593536 bytes 2026-02-18T06:41:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4366469120 bytes 2026-02-18T06:41:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4429724672 bytes 2026-02-18T06:41:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4451994624 bytes 2026-02-18T06:41:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4486853632 bytes 2026-02-18T06:41:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4534732800 bytes 2026-02-18T06:41:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4579218432 bytes 2026-02-18T06:41:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4619448320 bytes 2026-02-18T06:41:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4677106688 bytes 2026-02-18T06:41:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4778361856 bytes 2026-02-18T06:41:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4819277824 bytes 2026-02-18T06:41:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4895732736 bytes 2026-02-18T06:41:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4931749888 bytes 2026-02-18T06:41:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4977246208 bytes 2026-02-18T06:41:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5022089216 bytes 2026-02-18T06:41:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5096546304 bytes 2026-02-18T06:41:30.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5205614592 bytes 2026-02-18T06:41:31.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5282260992 bytes 2026-02-18T06:41:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5282852864 bytes 2026-02-18T06:41:31.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5363103744 bytes 2026-02-18T06:41:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5443710976 bytes 2026-02-18T06:41:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5474024448 bytes 2026-02-18T06:41:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5482958848 bytes 2026-02-18T06:41:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5510298624 bytes 2026-02-18T06:41:31.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5540740096 bytes 2026-02-18T06:41:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5590584320 bytes 2026-02-18T06:41:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5644975104 bytes 2026-02-18T06:41:31.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5668006912 bytes 2026-02-18T06:41:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5722937344 bytes 2026-02-18T06:41:31.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5830350848 bytes 2026-02-18T06:41:31.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5858725888 bytes 2026-02-18T06:41:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5920453632 bytes 2026-02-18T06:41:31.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5926218752 bytes 2026-02-18T06:41:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5932639232 bytes 2026-02-18T06:41:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5933369344 bytes 2026-02-18T06:41:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5971768320 bytes 2026-02-18T06:41:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6079073280 bytes 2026-02-18T06:41:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6128016384 bytes 2026-02-18T06:41:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6209984512 bytes 2026-02-18T06:41:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6300567552 bytes 2026-02-18T06:41:32.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6348189696 bytes 2026-02-18T06:41:32.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6380630016 bytes 2026-02-18T06:41:32.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6453344256 bytes 2026-02-18T06:41:32.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6534155264 bytes 2026-02-18T06:41:32.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6535539712 bytes 2026-02-18T06:41:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6650865664 bytes 2026-02-18T06:41:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6680822784 bytes 2026-02-18T06:41:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6722555904 bytes 2026-02-18T06:41:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6767644672 bytes 2026-02-18T06:41:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6851709952 bytes 2026-02-18T06:41:33.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6903705600 bytes 2026-02-18T06:41:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6913559552 bytes 2026-02-18T06:41:33.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6915602432 bytes 2026-02-18T06:41:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6916188160 bytes 2026-02-18T06:41:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7017633792 bytes 2026-02-18T06:41:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7035593728 bytes 2026-02-18T06:41:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7116561408 bytes 2026-02-18T06:41:33.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7152025600 bytes 2026-02-18T06:41:33.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7156128768 bytes 2026-02-18T06:41:33.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7196192768 bytes 2026-02-18T06:41:33.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7197033472 bytes 2026-02-18T06:41:33.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7232484352 bytes 2026-02-18T06:41:33.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7252347904 bytes 2026-02-18T06:41:33.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7306380288 bytes 2026-02-18T06:41:33.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7307003904 bytes 2026-02-18T06:41:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7394648064 bytes 2026-02-18T06:41:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7457359872 bytes 2026-02-18T06:41:33.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7464621056 bytes 2026-02-18T06:41:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7508850688 bytes 2026-02-18T06:41:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7569516544 bytes 2026-02-18T06:41:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7628452864 bytes 2026-02-18T06:41:34.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7644625920 bytes 2026-02-18T06:41:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7682093056 bytes 2026-02-18T06:41:34.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7727430656 bytes 2026-02-18T06:41:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7750479872 bytes 2026-02-18T06:41:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7843218432 bytes 2026-02-18T06:41:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7864933376 bytes 2026-02-18T06:41:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7883425792 bytes 2026-02-18T06:41:34.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7935915008 bytes 2026-02-18T06:41:34.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7989038080 bytes 2026-02-18T06:41:34.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8001940480 bytes 2026-02-18T06:41:34.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8085697536 bytes 2026-02-18T06:41:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8132722688 bytes 2026-02-18T06:41:34.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8164750336 bytes 2026-02-18T06:41:34.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8175812608 bytes 2026-02-18T06:41:34.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8218732544 bytes 2026-02-18T06:41:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8274418688 bytes 2026-02-18T06:41:34.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8288481280 bytes 2026-02-18T06:41:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8328912896 bytes 2026-02-18T06:41:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8329542656 bytes 2026-02-18T06:41:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8378335232 bytes 2026-02-18T06:41:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8417795072 bytes 2026-02-18T06:41:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8497771520 bytes 2026-02-18T06:41:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8498392064 bytes 2026-02-18T06:41:35.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8521796608 bytes 2026-02-18T06:41:35.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8551853056 bytes 2026-02-18T06:41:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8582315008 bytes 2026-02-18T06:41:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8643946496 bytes 2026-02-18T06:41:35.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8722145280 bytes 2026-02-18T06:41:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8773216256 bytes 2026-02-18T06:41:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8781289472 bytes 2026-02-18T06:41:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8833496064 bytes 2026-02-18T06:41:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8876427264 bytes 2026-02-18T06:41:35.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8903368704 bytes 2026-02-18T06:41:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8951033856 bytes 2026-02-18T06:41:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8973603840 bytes 2026-02-18T06:41:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8974194688 bytes 2026-02-18T06:41:35.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:41:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:41:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:41:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:41:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:41:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:41:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:41:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:41:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:41:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:41:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:41:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":17884501.33,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"} 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:41:07 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:41:08' 2026-02-18T06:41:36.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:41:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":17884501.33,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"} 2026-02-18T06:41:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:41:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:41:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:41:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:41:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:41:07' 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:41:07' 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:41:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:41:07' 2026-02-18T06:41:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:41:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:41:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:41:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK44944497mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":17884501.33,"bytes_per_snapshot":536322560.0,"last_snapshot_bytes":536535040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396829,"remote_snapshot_timestamp":1771396829,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:41:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:41:08' 2026-02-18T06:41:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:41:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:41:37.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:41:07' 2026-02-18T06:41:37.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:41:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:41:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:41:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:41:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:41:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap7 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap7 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7 2026-02-18T06:41:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:42:02.620 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T06:42:02.620 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T06:42:02.621 INFO:tasks.rbd_mirror_thrash:waiting for 1.3752897345514659 secs before reviving daemons 2026-02-18T06:42:02.621 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T06:42:02.620+0000 7f4800752640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 34397) UID: 1000 2026-02-18T06:42:03.997 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T06:42:03.997 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:42:03.998 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T06:42:03.998 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:42:03.998 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:42:04.000 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:42:04.000 INFO:tasks.rbd_mirror_thrash:waiting for 100.73757589415851 secs before thrashing 2026-02-18T06:42:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:42:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:42:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:42:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:42:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:06.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:42:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:42:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:42:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:42:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:42:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:42:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:42:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:42:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:42:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:42:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:42:07.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:42:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:42:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:42:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:42:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:42:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:42:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:42:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:42:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:42:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:42:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 23 2026-02-18T06:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:42:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:42:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:42:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:42:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:42:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:42:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:42:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:08.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:42:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:42:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=23 2026-02-18T06:42:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 23 = '' 2026-02-18T06:42:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:09.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:09.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 23 = '' 2026-02-18T06:42:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:10.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 23 = '' 2026-02-18T06:42:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:42:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=23 2026-02-18T06:42:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 23 = 23 2026-02-18T06:42:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:42:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:42:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-18T06:42:13.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 14 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:42:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:41:21 2026 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:42:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:42:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap6-clean ']' 2026-02-18T06:42:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6-clean 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:13.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap7 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:15.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap7-clean 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7-clean 2026-02-18T06:42:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:42:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:42:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:42:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:42:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 59277312 bytes 2026-02-18T06:42:16.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 132644864 bytes 2026-02-18T06:42:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 189187072 bytes 2026-02-18T06:42:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 265047040 bytes 2026-02-18T06:42:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 305483776 bytes 2026-02-18T06:42:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 343669760 bytes 2026-02-18T06:42:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 392192000 bytes 2026-02-18T06:42:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 425508864 bytes 2026-02-18T06:42:16.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 475856896 bytes 2026-02-18T06:42:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 505596928 bytes 2026-02-18T06:42:16.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 556564480 bytes 2026-02-18T06:42:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 596594688 bytes 2026-02-18T06:42:16.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 636104704 bytes 2026-02-18T06:42:16.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 637448192 bytes 2026-02-18T06:42:16.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 681754624 bytes 2026-02-18T06:42:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 688817152 bytes 2026-02-18T06:42:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 739757056 bytes 2026-02-18T06:42:17.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 797455360 bytes 2026-02-18T06:42:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 868313088 bytes 2026-02-18T06:42:17.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 949622784 bytes 2026-02-18T06:42:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 971798528 bytes 2026-02-18T06:42:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 995214336 bytes 2026-02-18T06:42:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1045099520 bytes 2026-02-18T06:42:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1093324800 bytes 2026-02-18T06:42:17.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1145197568 bytes 2026-02-18T06:42:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1176907776 bytes 2026-02-18T06:42:17.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1186568192 bytes 2026-02-18T06:42:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1234272256 bytes 2026-02-18T06:42:17.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1252706304 bytes 2026-02-18T06:42:17.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1308594176 bytes 2026-02-18T06:42:17.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1368763392 bytes 2026-02-18T06:42:17.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1403369472 bytes 2026-02-18T06:42:17.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1443331072 bytes 2026-02-18T06:42:18.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1458896896 bytes 2026-02-18T06:42:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1505208320 bytes 2026-02-18T06:42:18.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1527541760 bytes 2026-02-18T06:42:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1539118080 bytes 2026-02-18T06:42:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1595294720 bytes 2026-02-18T06:42:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1620065280 bytes 2026-02-18T06:42:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1687337984 bytes 2026-02-18T06:42:18.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1726144512 bytes 2026-02-18T06:42:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1767991296 bytes 2026-02-18T06:42:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1871395840 bytes 2026-02-18T06:42:18.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1893307392 bytes 2026-02-18T06:42:18.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1934852096 bytes 2026-02-18T06:42:18.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1992935424 bytes 2026-02-18T06:42:18.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2001480704 bytes 2026-02-18T06:42:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2030728192 bytes 2026-02-18T06:42:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2059199488 bytes 2026-02-18T06:42:18.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2148941824 bytes 2026-02-18T06:42:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2156991488 bytes 2026-02-18T06:42:18.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2176279552 bytes 2026-02-18T06:42:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2237381632 bytes 2026-02-18T06:42:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2298457088 bytes 2026-02-18T06:42:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2348101632 bytes 2026-02-18T06:42:19.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2402259968 bytes 2026-02-18T06:42:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2430432256 bytes 2026-02-18T06:42:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2471802880 bytes 2026-02-18T06:42:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2509501440 bytes 2026-02-18T06:42:19.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2579618816 bytes 2026-02-18T06:42:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2631689216 bytes 2026-02-18T06:42:19.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2726593536 bytes 2026-02-18T06:42:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2771487744 bytes 2026-02-18T06:42:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2812806144 bytes 2026-02-18T06:42:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2860386304 bytes 2026-02-18T06:42:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2887989248 bytes 2026-02-18T06:42:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2946534400 bytes 2026-02-18T06:42:19.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2973739008 bytes 2026-02-18T06:42:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2988942336 bytes 2026-02-18T06:42:19.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2998782976 bytes 2026-02-18T06:42:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3048501248 bytes 2026-02-18T06:42:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3049140224 bytes 2026-02-18T06:42:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3059910656 bytes 2026-02-18T06:42:20.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3127661568 bytes 2026-02-18T06:42:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3157870592 bytes 2026-02-18T06:42:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3173356544 bytes 2026-02-18T06:42:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3192697856 bytes 2026-02-18T06:42:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3193296896 bytes 2026-02-18T06:42:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3231697920 bytes 2026-02-18T06:42:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3274969088 bytes 2026-02-18T06:42:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3320285184 bytes 2026-02-18T06:42:20.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3357012992 bytes 2026-02-18T06:42:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3401043968 bytes 2026-02-18T06:42:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3420844032 bytes 2026-02-18T06:42:20.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3560388608 bytes 2026-02-18T06:42:20.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3683110912 bytes 2026-02-18T06:42:20.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3794239488 bytes 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:42:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:42:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:42:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:42:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:42:22.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536572928.0,"last_snapshot_bytes":536610816,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396875,"remote_snapshot_timestamp":1771396875,"replay_state":"idle"} 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.1 on trial051 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:42:07 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:42:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:42:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:42:08' 2026-02-18T06:42:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:42:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:42:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:42:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536572928.0,"last_snapshot_bytes":536610816,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396875,"remote_snapshot_timestamp":1771396875,"replay_state":"idle"} 2026-02-18T06:42:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:42:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:42:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:42:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:42:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:42:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.1 on trial051 2026-02-18T06:42:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:42:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:42:07' 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:42:07' 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:42:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:42:07' 2026-02-18T06:42:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:42:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:42:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:42:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142054278mirror.1trial05120.2.0-670-gadc19233trueOK42094279mirror.2trial05120.2.0-670-gadc19233falseOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK44944497mirror.3trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536572928.0,"last_snapshot_bytes":536610816,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396875,"remote_snapshot_timestamp":1771396875,"replay_state":"idle"}42054278mirror.1trial0512026-02-18 06:42:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:42:08' 2026-02-18T06:42:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:42:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:42:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:42:07' 2026-02-18T06:42:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:42:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:42:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:42:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:42:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap8 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap8 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8 2026-02-18T06:42:22.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:42:45.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3836563980267156) > weight (0.25) 2026-02-18T06:42:45.494 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T06:42:45.494 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T06:42:45.494 INFO:tasks.rbd_mirror_thrash:waiting for 4.013439528743644 secs before reviving daemons 2026-02-18T06:42:45.495 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:42:45.493+0000 7f374d857640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 29904) UID: 1000 2026-02-18T06:42:49.509 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T06:42:49.509 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:42:49.509 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T06:42:49.509 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T06:42:49.510 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T06:42:49.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T06:42:49.512 INFO:tasks.rbd_mirror_thrash:waiting for 100.90882985868974 secs before thrashing 2026-02-18T06:42:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:42:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:42:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:42:51.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:42:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:42:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:42:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:42:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:42:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:42:51.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:42:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:51.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:42:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:42:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:42:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:42:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:42:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:42:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:42:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:42:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:42:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:42:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:42:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:42:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:42:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:42:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:42:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:42:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:42:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:42:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:42:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:42:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:42:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:42:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:42:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:42:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:42:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:42:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:42:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:42:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:42:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:42:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:42:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 26 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:42:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:42:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:42:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:42:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:42:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:42:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:42:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:42:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:42:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:42:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=26 2026-02-18T06:42:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:53.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:54.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:54.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 26 = '' 2026-02-18T06:42:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 26 = '' 2026-02-18T06:42:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:42:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:55.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:42:55.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:42:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:42:55.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:42:55.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:42:55.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 26 = '' 2026-02-18T06:42:55.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:42:55.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:42:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:42:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:42:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:42:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:42:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=26 2026-02-18T06:42:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 26 = 26 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:42:57.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:42:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-18T06:42:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 16 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:42:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:41:21 2026 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:42:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:42:58.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap7-clean ']' 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7-clean 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:58.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap8 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:42:59.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:42:59.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:42:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:00.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:43:00.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap8-clean 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8-clean 2026-02-18T06:43:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:43:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:43:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:43:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:43:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:43:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:43:00.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:43:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 582656 bytes 2026-02-18T06:43:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 38315008 bytes 2026-02-18T06:43:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 77485056 bytes 2026-02-18T06:43:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 117927936 bytes 2026-02-18T06:43:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 162822144 bytes 2026-02-18T06:43:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 194939904 bytes 2026-02-18T06:43:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 217862144 bytes 2026-02-18T06:43:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 292923392 bytes 2026-02-18T06:43:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 312316928 bytes 2026-02-18T06:43:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 332761088 bytes 2026-02-18T06:43:01.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 368919552 bytes 2026-02-18T06:43:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 461192192 bytes 2026-02-18T06:43:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 506847232 bytes 2026-02-18T06:43:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 535526400 bytes 2026-02-18T06:43:01.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 593432576 bytes 2026-02-18T06:43:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 620017664 bytes 2026-02-18T06:43:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 654343168 bytes 2026-02-18T06:43:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 664341504 bytes 2026-02-18T06:43:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 681339904 bytes 2026-02-18T06:43:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 705725440 bytes 2026-02-18T06:43:02.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 791972864 bytes 2026-02-18T06:43:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 838552576 bytes 2026-02-18T06:43:02.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 860377088 bytes 2026-02-18T06:43:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 929555456 bytes 2026-02-18T06:43:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 934149120 bytes 2026-02-18T06:43:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1005059072 bytes 2026-02-18T06:43:02.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1028667392 bytes 2026-02-18T06:43:02.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1054372864 bytes 2026-02-18T06:43:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1122946048 bytes 2026-02-18T06:43:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1150494720 bytes 2026-02-18T06:43:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1177501696 bytes 2026-02-18T06:43:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1238308864 bytes 2026-02-18T06:43:02.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1288407040 bytes 2026-02-18T06:43:02.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1385283584 bytes 2026-02-18T06:43:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1401021440 bytes 2026-02-18T06:43:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1466142720 bytes 2026-02-18T06:43:03.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1477954560 bytes 2026-02-18T06:43:03.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1486191616 bytes 2026-02-18T06:43:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1535260672 bytes 2026-02-18T06:43:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1555065856 bytes 2026-02-18T06:43:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1569662976 bytes 2026-02-18T06:43:03.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1594609664 bytes 2026-02-18T06:43:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1617555456 bytes 2026-02-18T06:43:03.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1644375040 bytes 2026-02-18T06:43:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1650604032 bytes 2026-02-18T06:43:03.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1745253376 bytes 2026-02-18T06:43:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1748752384 bytes 2026-02-18T06:43:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1788073984 bytes 2026-02-18T06:43:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1800576000 bytes 2026-02-18T06:43:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1812815872 bytes 2026-02-18T06:43:03.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1854720000 bytes 2026-02-18T06:43:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1887426560 bytes 2026-02-18T06:43:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1924073472 bytes 2026-02-18T06:43:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1924678656 bytes 2026-02-18T06:43:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 1938245632 bytes 2026-02-18T06:43:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 1998454784 bytes 2026-02-18T06:43:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2042568704 bytes 2026-02-18T06:43:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2104368128 bytes 2026-02-18T06:43:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2177798144 bytes 2026-02-18T06:43:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2195840000 bytes 2026-02-18T06:43:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2206988288 bytes 2026-02-18T06:43:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2245260288 bytes 2026-02-18T06:43:04.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2309149696 bytes 2026-02-18T06:43:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2388454400 bytes 2026-02-18T06:43:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2428955648 bytes 2026-02-18T06:43:04.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2446292992 bytes 2026-02-18T06:43:04.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2485283840 bytes 2026-02-18T06:43:04.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2541336576 bytes 2026-02-18T06:43:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2572647424 bytes 2026-02-18T06:43:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2611274752 bytes 2026-02-18T06:43:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2674036736 bytes 2026-02-18T06:43:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2694634496 bytes 2026-02-18T06:43:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2709274624 bytes 2026-02-18T06:43:04.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2746380288 bytes 2026-02-18T06:43:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2848115712 bytes 2026-02-18T06:43:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2865118208 bytes 2026-02-18T06:43:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2895458304 bytes 2026-02-18T06:43:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2941059072 bytes 2026-02-18T06:43:05.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2995032064 bytes 2026-02-18T06:43:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3030113280 bytes 2026-02-18T06:43:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3092694016 bytes 2026-02-18T06:43:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3107502080 bytes 2026-02-18T06:43:05.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3130438656 bytes 2026-02-18T06:43:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3155119104 bytes 2026-02-18T06:43:05.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3187332096 bytes 2026-02-18T06:43:05.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3234002944 bytes 2026-02-18T06:43:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3278164992 bytes 2026-02-18T06:43:05.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3286867968 bytes 2026-02-18T06:43:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3355791360 bytes 2026-02-18T06:43:05.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3356401664 bytes 2026-02-18T06:43:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3364345856 bytes 2026-02-18T06:43:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3460026368 bytes 2026-02-18T06:43:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3531432960 bytes 2026-02-18T06:43:05.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3540759552 bytes 2026-02-18T06:43:05.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3557673984 bytes 2026-02-18T06:43:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3610202112 bytes 2026-02-18T06:43:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3612553216 bytes 2026-02-18T06:43:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3700261888 bytes 2026-02-18T06:43:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3783351296 bytes 2026-02-18T06:43:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3857259520 bytes 2026-02-18T06:43:06.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3873816576 bytes 2026-02-18T06:43:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3935243264 bytes 2026-02-18T06:43:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3935883264 bytes 2026-02-18T06:43:06.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4009410560 bytes 2026-02-18T06:43:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4069980160 bytes 2026-02-18T06:43:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4102138880 bytes 2026-02-18T06:43:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4141020160 bytes 2026-02-18T06:43:06.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4169468928 bytes 2026-02-18T06:43:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4197586944 bytes 2026-02-18T06:43:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4205712384 bytes 2026-02-18T06:43:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4210694144 bytes 2026-02-18T06:43:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4219067392 bytes 2026-02-18T06:43:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4264224768 bytes 2026-02-18T06:43:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4301684736 bytes 2026-02-18T06:43:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4354238464 bytes 2026-02-18T06:43:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4397581312 bytes 2026-02-18T06:43:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4408866816 bytes 2026-02-18T06:43:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4471421952 bytes 2026-02-18T06:43:07.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4507824128 bytes 2026-02-18T06:43:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4521433088 bytes 2026-02-18T06:43:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4604670976 bytes 2026-02-18T06:43:07.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4628312064 bytes 2026-02-18T06:43:07.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4656873472 bytes 2026-02-18T06:43:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4699671552 bytes 2026-02-18T06:43:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4753947648 bytes 2026-02-18T06:43:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 4785814528 bytes 2026-02-18T06:43:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 4845341696 bytes 2026-02-18T06:43:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 4883497984 bytes 2026-02-18T06:43:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 4947012608 bytes 2026-02-18T06:43:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5030088704 bytes 2026-02-18T06:43:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5076898816 bytes 2026-02-18T06:43:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5077506048 bytes 2026-02-18T06:43:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5134094336 bytes 2026-02-18T06:43:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5213932544 bytes 2026-02-18T06:43:08.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5277412352 bytes 2026-02-18T06:43:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5321656320 bytes 2026-02-18T06:43:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5345495040 bytes 2026-02-18T06:43:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5367704576 bytes 2026-02-18T06:43:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5368324096 bytes 2026-02-18T06:43:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5401624576 bytes 2026-02-18T06:43:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5405030400 bytes 2026-02-18T06:43:08.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5496415232 bytes 2026-02-18T06:43:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5566291968 bytes 2026-02-18T06:43:08.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5624728576 bytes 2026-02-18T06:43:08.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5687546880 bytes 2026-02-18T06:43:08.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5732179968 bytes 2026-02-18T06:43:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5804619776 bytes 2026-02-18T06:43:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5822592000 bytes 2026-02-18T06:43:08.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5846642688 bytes 2026-02-18T06:43:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 5908616192 bytes 2026-02-18T06:43:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 5977826304 bytes 2026-02-18T06:43:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6025821184 bytes 2026-02-18T06:43:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6041713664 bytes 2026-02-18T06:43:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6065318912 bytes 2026-02-18T06:43:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6065935360 bytes 2026-02-18T06:43:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6098540544 bytes 2026-02-18T06:43:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6143032320 bytes 2026-02-18T06:43:09.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6197302272 bytes 2026-02-18T06:43:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6243385344 bytes 2026-02-18T06:43:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6247780352 bytes 2026-02-18T06:43:09.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6268676096 bytes 2026-02-18T06:43:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6269277184 bytes 2026-02-18T06:43:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6291331072 bytes 2026-02-18T06:43:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6357501952 bytes 2026-02-18T06:43:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6397998080 bytes 2026-02-18T06:43:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6466894848 bytes 2026-02-18T06:43:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6592644096 bytes 2026-02-18T06:43:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6649468928 bytes 2026-02-18T06:43:10.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6710290432 bytes 2026-02-18T06:43:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6781379584 bytes 2026-02-18T06:43:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6850168832 bytes 2026-02-18T06:43:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6882935808 bytes 2026-02-18T06:43:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6929918976 bytes 2026-02-18T06:43:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6951364608 bytes 2026-02-18T06:43:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 6991097856 bytes 2026-02-18T06:43:10.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7090835456 bytes 2026-02-18T06:43:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7161325568 bytes 2026-02-18T06:43:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7177358336 bytes 2026-02-18T06:43:10.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7217337344 bytes 2026-02-18T06:43:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7270152192 bytes 2026-02-18T06:43:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7275345920 bytes 2026-02-18T06:43:10.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7313799168 bytes 2026-02-18T06:43:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7325009920 bytes 2026-02-18T06:43:10.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7340723200 bytes 2026-02-18T06:43:10.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7357135872 bytes 2026-02-18T06:43:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7447639040 bytes 2026-02-18T06:43:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7480708096 bytes 2026-02-18T06:43:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7512468480 bytes 2026-02-18T06:43:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7566607360 bytes 2026-02-18T06:43:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7589445632 bytes 2026-02-18T06:43:11.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7597133824 bytes 2026-02-18T06:43:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7661996032 bytes 2026-02-18T06:43:11.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7687101440 bytes 2026-02-18T06:43:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7706099712 bytes 2026-02-18T06:43:11.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7766538240 bytes 2026-02-18T06:43:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7871026176 bytes 2026-02-18T06:43:11.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7929647104 bytes 2026-02-18T06:43:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 7930745856 bytes 2026-02-18T06:43:11.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 7971102720 bytes 2026-02-18T06:43:11.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8032326656 bytes 2026-02-18T06:43:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8068516864 bytes 2026-02-18T06:43:11.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8158115840 bytes 2026-02-18T06:43:11.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8186397696 bytes 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:43:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:43:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:43:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:43:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:43:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:43:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":48787269.82,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"} 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.2 on trial051 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:43:07 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:43:08' 2026-02-18T06:43:13.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:43:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:43:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:43:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":48787269.82,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"} 2026-02-18T06:43:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:43:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:43:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:43:13.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:43:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:43:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.2 on trial051 2026-02-18T06:43:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:43:07' 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:43:07' 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:43:07' 2026-02-18T06:43:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:43:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:43:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:43:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142094279mirror.2trial05120.2.0-670-gadc19233trueOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK44944497mirror.3trial05120.2.0-670-gadc19233falseOK45394542mirror.1trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":48787269.82,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"}42094279mirror.2trial0512026-02-18 06:43:07cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:43:08' 2026-02-18T06:43:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:43:13.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:43:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:43:07' 2026-02-18T06:43:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:43:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:43:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:43:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:43:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:43:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap9 2026-02-18T06:43:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap9 2026-02-18T06:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9 2026-02-18T06:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:43:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:43:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:41.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:43:41.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:43:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:43:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:43:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:43:42.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:43:42.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:43:42.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:43:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:43:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:43:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:43:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:43:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:43:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:43:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:43:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:43:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:43:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:43:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:43:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:43:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:43:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:43:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:43:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:43:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:43:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:43:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:43:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:43:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:43:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:43:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:43:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:43:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:43:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:43:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:43:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:43:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:43:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:43:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:43:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:43:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:43:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:43:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:43:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:43:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:43:44.739 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7522620831698755) > weight (0.25) 2026-02-18T06:43:44.739 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9751298504193632) > weight (0.25) 2026-02-18T06:43:44.739 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7758365180229204) > weight (0.25) 2026-02-18T06:43:44.739 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9645192955964539) > weight (0.25) 2026-02-18T06:43:44.739 INFO:tasks.rbd_mirror_thrash:waiting for 75.62381675050116 secs before thrashing 2026-02-18T06:43:44.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 29 2026-02-18T06:43:44.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:43:44.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:43:45.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=29 2026-02-18T06:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:43:45.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:43:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:43:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:43:45.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:43:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:43:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:43:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:43:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:43:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 29 = '' 2026-02-18T06:43:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:43:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:43:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:43:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:43:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:43:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:43:45.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:43:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:43:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:43:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:43:45.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 29 = '' 2026-02-18T06:43:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:43:45.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:43:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:43:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:43:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:43:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:43:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:43:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:43:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:43:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:43:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 29 = '' 2026-02-18T06:43:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:43:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:43:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:43:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:43:48.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=29 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 29 = 29 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:43:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9 2026-02-18T06:43:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:43:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:43:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:43:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-18T06:43:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:43:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:43:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 18 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:42:58 2026 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:43:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:43:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:43:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:43:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:43:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:43:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap8-clean ']' 2026-02-18T06:43:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8-clean 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:43:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:43:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:50.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:43:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:50.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap9 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:43:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:50.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:50.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:43:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:43:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:43:51.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:43:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:43:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap9-clean 2026-02-18T06:43:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-18T06:43:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9-clean 2026-02-18T06:43:51.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror test 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:43:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:43:51.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:43:51.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:43:51.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-18T06:43:52.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 44284928 bytes 2026-02-18T06:43:52.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 45143040 bytes 2026-02-18T06:43:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 98920448 bytes 2026-02-18T06:43:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 143060992 bytes 2026-02-18T06:43:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 188236800 bytes 2026-02-18T06:43:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 190101504 bytes 2026-02-18T06:43:52.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 276876288 bytes 2026-02-18T06:43:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 336393216 bytes 2026-02-18T06:43:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 405576704 bytes 2026-02-18T06:43:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 485352448 bytes 2026-02-18T06:43:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 519507968 bytes 2026-02-18T06:43:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 567563264 bytes 2026-02-18T06:43:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 642928640 bytes 2026-02-18T06:43:52.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 651108352 bytes 2026-02-18T06:43:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 671471616 bytes 2026-02-18T06:43:53.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 756939776 bytes 2026-02-18T06:43:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 786426880 bytes 2026-02-18T06:43:53.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 843467776 bytes 2026-02-18T06:43:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 922782720 bytes 2026-02-18T06:43:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 969275392 bytes 2026-02-18T06:43:53.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1002768384 bytes 2026-02-18T06:43:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1100554240 bytes 2026-02-18T06:43:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1101137920 bytes 2026-02-18T06:43:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1168143360 bytes 2026-02-18T06:43:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1194839040 bytes 2026-02-18T06:43:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1291522048 bytes 2026-02-18T06:43:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1345353728 bytes 2026-02-18T06:43:53.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1424411648 bytes 2026-02-18T06:43:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1445225472 bytes 2026-02-18T06:43:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1471601664 bytes 2026-02-18T06:43:53.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1511999488 bytes 2026-02-18T06:43:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1571619840 bytes 2026-02-18T06:43:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1685579776 bytes 2026-02-18T06:43:54.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1774638080 bytes 2026-02-18T06:43:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1870513152 bytes 2026-02-18T06:43:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1933190144 bytes 2026-02-18T06:43:54.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1975659520 bytes 2026-02-18T06:43:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 2016845824 bytes 2026-02-18T06:43:54.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 2073492480 bytes 2026-02-18T06:43:54.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2100633600 bytes 2026-02-18T06:43:54.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2101222400 bytes 2026-02-18T06:43:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2119888896 bytes 2026-02-18T06:43:54.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2159329280 bytes 2026-02-18T06:43:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2210122752 bytes 2026-02-18T06:43:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2232497152 bytes 2026-02-18T06:43:54.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2252046336 bytes 2026-02-18T06:43:54.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2278280192 bytes 2026-02-18T06:43:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2318523392 bytes 2026-02-18T06:43:55.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2417661952 bytes 2026-02-18T06:43:55.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2475745280 bytes 2026-02-18T06:43:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2555580416 bytes 2026-02-18T06:43:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2599327744 bytes 2026-02-18T06:43:55.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2710367232 bytes 2026-02-18T06:43:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2744592384 bytes 2026-02-18T06:43:55.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2796000256 bytes 2026-02-18T06:43:55.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2810220544 bytes 2026-02-18T06:43:55.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2861085696 bytes 2026-02-18T06:43:55.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2887341056 bytes 2026-02-18T06:43:55.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2966309888 bytes 2026-02-18T06:43:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2996773888 bytes 2026-02-18T06:43:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 3029033984 bytes 2026-02-18T06:43:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 3090614272 bytes 2026-02-18T06:43:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 3110146048 bytes 2026-02-18T06:43:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3146168320 bytes 2026-02-18T06:43:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3146791936 bytes 2026-02-18T06:43:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3202483200 bytes 2026-02-18T06:43:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3227321344 bytes 2026-02-18T06:43:56.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3271044096 bytes 2026-02-18T06:43:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3293661184 bytes 2026-02-18T06:43:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3367515136 bytes 2026-02-18T06:43:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3405728768 bytes 2026-02-18T06:43:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3422787584 bytes 2026-02-18T06:43:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3473883136 bytes 2026-02-18T06:43:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3521886208 bytes 2026-02-18T06:43:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3582006272 bytes 2026-02-18T06:43:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3585599488 bytes 2026-02-18T06:43:56.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3671526400 bytes 2026-02-18T06:43:56.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3748038656 bytes 2026-02-18T06:43:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3794667520 bytes 2026-02-18T06:43:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3845384192 bytes 2026-02-18T06:43:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3885648896 bytes 2026-02-18T06:43:57.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3923258368 bytes 2026-02-18T06:43:57.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3979773952 bytes 2026-02-18T06:43:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 4007604224 bytes 2026-02-18T06:43:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 4037880832 bytes 2026-02-18T06:43:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4145559552 bytes 2026-02-18T06:43:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4200834048 bytes 2026-02-18T06:43:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4222772224 bytes 2026-02-18T06:43:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4225491968 bytes 2026-02-18T06:43:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4273221632 bytes 2026-02-18T06:43:57.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4316820480 bytes 2026-02-18T06:43:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4351780864 bytes 2026-02-18T06:43:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4379254784 bytes 2026-02-18T06:43:57.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4436257792 bytes 2026-02-18T06:43:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4505081856 bytes 2026-02-18T06:43:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4574512128 bytes 2026-02-18T06:43:57.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4575132672 bytes 2026-02-18T06:43:58.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4651131904 bytes 2026-02-18T06:43:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4669976576 bytes 2026-02-18T06:43:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4746966016 bytes 2026-02-18T06:43:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4771432448 bytes 2026-02-18T06:43:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4800870400 bytes 2026-02-18T06:43:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4807165952 bytes 2026-02-18T06:43:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4868144128 bytes 2026-02-18T06:43:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4889392128 bytes 2026-02-18T06:43:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4912301056 bytes 2026-02-18T06:43:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5005757440 bytes 2026-02-18T06:43:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5024902144 bytes 2026-02-18T06:43:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5071351808 bytes 2026-02-18T06:43:58.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5147254784 bytes 2026-02-18T06:43:58.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5220503552 bytes 2026-02-18T06:43:58.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5249086464 bytes 2026-02-18T06:43:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5306968064 bytes 2026-02-18T06:43:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5340237824 bytes 2026-02-18T06:43:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5435276288 bytes 2026-02-18T06:43:59.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5448115200 bytes 2026-02-18T06:43:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5534931968 bytes 2026-02-18T06:43:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5562128384 bytes 2026-02-18T06:43:59.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5612054528 bytes 2026-02-18T06:43:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5676797952 bytes 2026-02-18T06:43:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5695849472 bytes 2026-02-18T06:43:59.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5735498752 bytes 2026-02-18T06:43:59.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5772796928 bytes 2026-02-18T06:43:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5781925888 bytes 2026-02-18T06:43:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5803553792 bytes 2026-02-18T06:43:59.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5869009920 bytes 2026-02-18T06:43:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5869633536 bytes 2026-02-18T06:43:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5944371200 bytes 2026-02-18T06:43:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5955126272 bytes 2026-02-18T06:44:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 6001523712 bytes 2026-02-18T06:44:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 6021935104 bytes 2026-02-18T06:44:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 6072084480 bytes 2026-02-18T06:44:00.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6077535232 bytes 2026-02-18T06:44:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6112730112 bytes 2026-02-18T06:44:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6149086208 bytes 2026-02-18T06:44:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6187235328 bytes 2026-02-18T06:44:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6220265472 bytes 2026-02-18T06:44:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6282519552 bytes 2026-02-18T06:44:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6307470336 bytes 2026-02-18T06:44:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6338553856 bytes 2026-02-18T06:44:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6345077760 bytes 2026-02-18T06:44:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6410451968 bytes 2026-02-18T06:44:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6491524096 bytes 2026-02-18T06:44:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6501750784 bytes 2026-02-18T06:44:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6522571776 bytes 2026-02-18T06:44:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6553150464 bytes 2026-02-18T06:44:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6572428288 bytes 2026-02-18T06:44:00.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6594158592 bytes 2026-02-18T06:44:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6662221824 bytes 2026-02-18T06:44:01.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6713438208 bytes 2026-02-18T06:44:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6721980416 bytes 2026-02-18T06:44:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6793105408 bytes 2026-02-18T06:44:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6875420672 bytes 2026-02-18T06:44:01.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6941642752 bytes 2026-02-18T06:44:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6944152576 bytes 2026-02-18T06:44:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 7003843584 bytes 2026-02-18T06:44:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7028282368 bytes 2026-02-18T06:44:01.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7110320128 bytes 2026-02-18T06:44:01.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7156113408 bytes 2026-02-18T06:44:01.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7175458816 bytes 2026-02-18T06:44:01.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7220208640 bytes 2026-02-18T06:44:01.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7225406464 bytes 2026-02-18T06:44:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7274276864 bytes 2026-02-18T06:44:01.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7317097472 bytes 2026-02-18T06:44:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7359945728 bytes 2026-02-18T06:44:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7435659264 bytes 2026-02-18T06:44:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7547196416 bytes 2026-02-18T06:44:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7574213632 bytes 2026-02-18T06:44:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7599365120 bytes 2026-02-18T06:44:02.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7644560384 bytes 2026-02-18T06:44:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7645156352 bytes 2026-02-18T06:44:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7686451200 bytes 2026-02-18T06:44:02.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7729607680 bytes 2026-02-18T06:44:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7764709376 bytes 2026-02-18T06:44:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7800120320 bytes 2026-02-18T06:44:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7840611328 bytes 2026-02-18T06:44:02.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7884456960 bytes 2026-02-18T06:44:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7897945088 bytes 2026-02-18T06:44:02.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7949492224 bytes 2026-02-18T06:44:02.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7997571072 bytes 2026-02-18T06:44:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 8072264704 bytes 2026-02-18T06:44:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 8141507584 bytes 2026-02-18T06:44:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 8151527424 bytes 2026-02-18T06:44:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 8200622080 bytes 2026-02-18T06:44:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8311118848 bytes 2026-02-18T06:44:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8354983936 bytes 2026-02-18T06:44:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8407063552 bytes 2026-02-18T06:44:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8431333376 bytes 2026-02-18T06:44:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8509321216 bytes 2026-02-18T06:44:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8527292416 bytes 2026-02-18T06:44:03.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8558013440 bytes 2026-02-18T06:44:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8589164544 bytes 2026-02-18T06:44:03.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8660040704 bytes 2026-02-18T06:44:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8728079360 bytes 2026-02-18T06:44:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8772630528 bytes 2026-02-18T06:44:03.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8810062848 bytes 2026-02-18T06:44:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8834437120 bytes 2026-02-18T06:44:03.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8855071744 bytes 2026-02-18T06:44:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8886789120 bytes 2026-02-18T06:44:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8957720576 bytes 2026-02-18T06:44:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 9024552960 bytes 2026-02-18T06:44:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 9115540480 bytes 2026-02-18T06:44:04.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 9128020992 bytes 2026-02-18T06:44:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 9185325056 bytes 2026-02-18T06:44:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 9252532224 bytes 2026-02-18T06:44:04.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 9300244480 bytes 2026-02-18T06:44:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 9338910720 bytes 2026-02-18T06:44:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 9352478720 bytes 2026-02-18T06:44:04.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 9448345600 bytes 2026-02-18T06:44:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 9495308288 bytes 2026-02-18T06:44:04.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 9522327552 bytes 2026-02-18T06:44:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 9564695552 bytes 2026-02-18T06:44:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 9611044864 bytes 2026-02-18T06:44:04.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 9659158528 bytes 2026-02-18T06:44:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 9665029120 bytes 2026-02-18T06:44:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 9750090752 bytes 2026-02-18T06:44:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 9830099968 bytes 2026-02-18T06:44:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 9892737024 bytes 2026-02-18T06:44:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 9909679104 bytes 2026-02-18T06:44:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 9945524224 bytes 2026-02-18T06:44:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 9957782528 bytes 2026-02-18T06:44:05.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 10021485568 bytes 2026-02-18T06:44:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 10081441792 bytes 2026-02-18T06:44:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 10171588608 bytes 2026-02-18T06:44:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 10216268800 bytes 2026-02-18T06:44:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 10276639744 bytes 2026-02-18T06:44:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 10314573824 bytes 2026-02-18T06:44:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 10329837568 bytes 2026-02-18T06:44:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 10359454720 bytes 2026-02-18T06:44:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23000 IOs, 10397462528 bytes 2026-02-18T06:44:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23100 IOs, 10458332160 bytes 2026-02-18T06:44:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23200 IOs, 10529841152 bytes 2026-02-18T06:44:05.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23300 IOs, 10580108288 bytes 2026-02-18T06:44:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23400 IOs, 10580700160 bytes 2026-02-18T06:44:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:44:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:44:05.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:44:05.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:44:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-18T06:44:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local description_pattern= 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local service_pattern= 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:44:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-18T06:44:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='test: 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: global_id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+replaying 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"} 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: service: mirror.2 on trial051 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:43:37 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: peer_sites: 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: name: cluster2 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: state: up+stopped 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: description: local image is primary 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: last_update: 2026-02-18 06:43:38' 2026-02-18T06:44:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ state: .*up+replaying' 2026-02-18T06:44:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: state: up+replaying 2026-02-18T06:44:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^ description: .*' 2026-02-18T06:44:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"} 2026-02-18T06:44:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:44:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:44:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep '^up+' 2026-02-18T06:44:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:44:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep 'service: *mirror..* on ' 2026-02-18T06:44:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: service: mirror.2 on trial051 2026-02-18T06:44:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update 2026-02-18T06:44:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-18T06:44:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:43:37' 2026-02-18T06:44:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-18 06:43:37' 2026-02-18T06:44:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state_pattern=up+replaying 2026-02-18T06:44:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local 'prev_last_update=2026-02-18 06:43:37' 2026-02-18T06:44:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status 2026-02-18T06:44:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ CEPH_ARGS= 2026-02-18T06:44:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-18T06:44:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status='OKOKOK142094279mirror.2trial05120.2.0-670-gadc19233trueOK43244327mirror.0trial05120.2.0-670-gadc19233falseOK44944497mirror.3trial05120.2.0-670-gadc19233falseOK45394542mirror.1trial05120.2.0-670-gadc19233falseOKtest08155eb4-ae3a-4e60-a23b-30e025844809up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536659968.0,"last_snapshot_bytes":536659968,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771396972,"remote_snapshot_timestamp":1771396972,"replay_state":"idle"}42094279mirror.2trial0512026-02-18 06:43:37cluster219836290-025c-408f-8a50-82436ee28aa0up+stoppedlocal image is primary2026-02-18 06:43:38' 2026-02-18T06:44:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_update state 2026-02-18T06:44:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-18T06:44:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ last_update='2026-02-18 06:43:37' 2026-02-18T06:44:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-18T06:44:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=up+replaying 2026-02-18T06:44:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo up+replaying 2026-02-18T06:44:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep up+replaying 2026-02-18T06:44:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:up+replaying 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap10 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap10 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10 2026-02-18T06:44:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:44:30.421 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T06:44:30.422 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T06:44:30.422 INFO:tasks.rbd_mirror_thrash:waiting for 6.900265618649492 secs before reviving daemons 2026-02-18T06:44:30.423 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:44:30.421+0000 7fa2cdbcb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 32197) UID: 1000 2026-02-18T06:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:44:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:44:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:44:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:44:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:44:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:44:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:44:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:44:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:44:37.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:37.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:37.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:37.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:44:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:44:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:44:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:44:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:44:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:44:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:44:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:44:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:44:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:44:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:44:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:44:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:44:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:44:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:44:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:44:37.324 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T06:44:37.324 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:44:37.324 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T06:44:37.324 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T06:44:37.325 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T06:44:37.328 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T06:44:37.328 INFO:tasks.rbd_mirror_thrash:waiting for 119.53808834022493 secs before thrashing 2026-02-18T06:44:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 32 2026-02-18T06:44:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:44:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:44:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:44:38.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:44:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:44:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:38.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:44:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=32 2026-02-18T06:44:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:44:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:39.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:39.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:39.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:44:39.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:44:39.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:44:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:44:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 32 = '' 2026-02-18T06:44:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:44:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:44:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:44:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:44:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 32 = '' 2026-02-18T06:44:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:44:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:44:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:44:40.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:44:40.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 32 = '' 2026-02-18T06:44:40.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:40.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:42.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=32 2026-02-18T06:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 32 = 32 2026-02-18T06:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-18T06:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:44:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:44:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-18T06:44:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:44:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:44:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:44:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 20 2026-02-18T06:44:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:42:58 2026 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:44:43.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:44:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:44:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:44:43.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:44:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snap9-clean ']' 2026-02-18T06:44:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-18T06:44:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9-clean 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:43.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap10 2026-02-18T06:44:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10 2026-02-18T06:44:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:44.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:45.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:45.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ clean_snap_name=snap10-clean 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10-clean 2026-02-18T06:44:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-18T06:44:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-18T06:44:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:45.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:44:45.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:44:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:44:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:44:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:44:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:44:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:44:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:44:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:44:46.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-18T06:44:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:44:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:44:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:44:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:44:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:44:47.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:44:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-18T06:44:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-18T06:44:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:44:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:44:47.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:44:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:44:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:44:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-18T06:44:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:44:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:44:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:44:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:44:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-18T06:44:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 34 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:44:48.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-test.status 2026-02-18T06:44:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-test.status 2026-02-18T06:44:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:44:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-test.status 2026-02-18T06:44:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-test.status 2026-02-18T06:44:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.status 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-18T06:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:49.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:44:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=34 2026-02-18T06:44:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:44:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:44:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:44:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:44:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:44:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 34 = '' 2026-02-18T06:44:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:44:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:49.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:49.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:49.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:49.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:49.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:44:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:44:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:44:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:44:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:44:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 34 = '' 2026-02-18T06:44:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:44:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:44:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.status 2026-02-18T06:44:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-18T06:44:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:44:50.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:44:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=34 2026-02-18T06:44:50.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 34 = 34 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10-clean 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:44:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:44:51.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-18T06:44:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'test': 2026-02-18T06:44:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T06:44:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:44:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 21 2026-02-18T06:44:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b6706cf9ce 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b6706cf9ce 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:44:43 2026 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:36:39 2026 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:44:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:44:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:44:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 08155eb4-ae3a-4e60-a23b-30e025844809 2026-02-18T06:44:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:44:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 10 2026-02-18T06:44:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap1 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap1 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:52.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:52.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:52.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap1-clean 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap1-clean 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:52.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:52.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:53.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:54.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap2 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap2 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2 2026-02-18T06:44:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:54.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap2-clean 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap2-clean 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:55.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:56.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap3 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap3 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:56.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:57.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:57.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap3-clean 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap3-clean 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:57.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:57.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:57.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:58.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:44:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:44:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap4 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap4 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:44:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:44:59.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:44:59.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:00.364 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T06:45:00.364 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T06:45:00.364 INFO:tasks.rbd_mirror_thrash:waiting for 4.507253816759281 secs before reviving daemons 2026-02-18T06:45:00.365 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T06:45:00.363+0000 7fe2e129b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 35415) UID: 1000 2026-02-18T06:45:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap4-clean 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap4-clean 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:00.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:00.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:01.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap5 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap5 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:01.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:01.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:02.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:02.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap5-clean 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap5-clean 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:03.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:03.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:03.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap6 2026-02-18T06:45:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap6 2026-02-18T06:45:04.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:04.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:04.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:04.873 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T06:45:04.873 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:45:04.873 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T06:45:04.873 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:45:04.874 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:45:04.876 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:45:04.876 INFO:tasks.rbd_mirror_thrash:waiting for 110.0628906633838 secs before thrashing 2026-02-18T06:45:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:04.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:05.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap6-clean 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap6-clean 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:05.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:05.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:05.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:06.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap7 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap7 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:07.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:07.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap7-clean 2026-02-18T06:45:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-18T06:45:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap7-clean 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:08.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap8 2026-02-18T06:45:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap8 2026-02-18T06:45:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:09.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:09.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap8-clean 2026-02-18T06:45:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap8-clean 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap9 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap9 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:11.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:12.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:12.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:12.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:12.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap9-clean 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap9-clean 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:12.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:12.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap10 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap10 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap10-clean 2026-02-18T06:45:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap10-clean 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export 2026-02-18T06:45:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-18T06:45:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-test.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-test.export 2026-02-18T06:45:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:16.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 10 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap1 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap1 2026-02-18T06:45:16.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-18T06:45:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap1-clean 2026-02-18T06:45:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-18T06:45:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap1-clean 2026-02-18T06:45:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-18T06:45:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap2 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap2 2026-02-18T06:45:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-18T06:45:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap2-clean 2026-02-18T06:45:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-18T06:45:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap2-clean 2026-02-18T06:45:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-18T06:45:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap3 2026-02-18T06:45:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-18T06:45:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap3 2026-02-18T06:45:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-18T06:45:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap3-clean 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap3-clean 2026-02-18T06:45:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-18T06:45:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap4 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap4 2026-02-18T06:45:22.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-18T06:45:23.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap4-clean 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap4-clean 2026-02-18T06:45:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-18T06:45:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:24.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:24.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap5 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap5 2026-02-18T06:45:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-18T06:45:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap5-clean 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap5-clean 2026-02-18T06:45:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-18T06:45:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap6 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap6 2026-02-18T06:45:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-18T06:45:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap6-clean 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap6-clean 2026-02-18T06:45:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-18T06:45:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap7 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap7 2026-02-18T06:45:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-18T06:45:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap7-clean 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap7-clean 2026-02-18T06:45:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-18T06:45:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap8 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap8 2026-02-18T06:45:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-18T06:45:31.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap8-clean 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap8-clean 2026-02-18T06:45:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-18T06:45:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:32.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap9 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap9 2026-02-18T06:45:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-18T06:45:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap9-clean 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap9-clean 2026-02-18T06:45:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-18T06:45:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T06:45:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap10 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap10 2026-02-18T06:45:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-18T06:45:34.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap10-clean 2026-02-18T06:45:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-18T06:45:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap10-clean 2026-02-18T06:45:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-18T06:45:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T06:45:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror test 2026-02-18T06:45:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T06:45:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T06:45:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror test 2026-02-18T06:45:35.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:35.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-18T06:45:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-18T06:45:36.048 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T06:45:36.046+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d274704e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T06:45:36.049 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T06:45:36.046+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27299900 [3/08155eb4-ae3a-4e60-a23b-30e025844809] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T06:45:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror test 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=test 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-18T06:45:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T06:45:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-18T06:45:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T06:45:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T06:45:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=test 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T06:45:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror test 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=test 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-18T06:45:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T06:45:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: create many images' 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T06:45:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T06:45:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 06:45:37 TEST: create many images 2026-02-18T06:45:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 06:45:37 TEST: create many images 2026-02-18T06:45:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ snap_name=snap 2026-02-18T06:45:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 50 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_1 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:45:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:45:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_1 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:45:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-18T06:45:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:45:38.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-18T06:45:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_1': 2026-02-18T06:45:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:45:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 127ae51d3842 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.127ae51d3842 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:45:37 2026 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:45:37 2026 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:45:37 2026 2026-02-18T06:45:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:45:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:45:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1f218227-0cf6-470c-8df3-a73ed823e755 2026-02-18T06:45:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:45:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:45:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:45:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:45:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:45:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 23605248 bytes 2026-02-18T06:45:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 81569792 bytes 2026-02-18T06:45:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 112742400 bytes 2026-02-18T06:45:38.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 113368064 bytes 2026-02-18T06:45:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 130457600 bytes 2026-02-18T06:45:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 131053568 bytes 2026-02-18T06:45:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 220527616 bytes 2026-02-18T06:45:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 284453888 bytes 2026-02-18T06:45:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 305602560 bytes 2026-02-18T06:45:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 345260032 bytes 2026-02-18T06:45:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 358379520 bytes 2026-02-18T06:45:38.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 401966080 bytes 2026-02-18T06:45:38.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 427681792 bytes 2026-02-18T06:45:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 488517632 bytes 2026-02-18T06:45:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 579709952 bytes 2026-02-18T06:45:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 642769920 bytes 2026-02-18T06:45:39.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 675324928 bytes 2026-02-18T06:45:39.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 717872128 bytes 2026-02-18T06:45:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 740483072 bytes 2026-02-18T06:45:39.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 741733376 bytes 2026-02-18T06:45:39.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 808573952 bytes 2026-02-18T06:45:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 809191424 bytes 2026-02-18T06:45:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 869288960 bytes 2026-02-18T06:45:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 892313600 bytes 2026-02-18T06:45:39.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 938365952 bytes 2026-02-18T06:45:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 947709952 bytes 2026-02-18T06:45:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 969379840 bytes 2026-02-18T06:45:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 981394432 bytes 2026-02-18T06:45:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 984580096 bytes 2026-02-18T06:45:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1030635520 bytes 2026-02-18T06:45:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1081936896 bytes 2026-02-18T06:45:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1185855488 bytes 2026-02-18T06:45:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1296229376 bytes 2026-02-18T06:45:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1315219456 bytes 2026-02-18T06:45:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1353797632 bytes 2026-02-18T06:45:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1428833280 bytes 2026-02-18T06:45:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1429455872 bytes 2026-02-18T06:45:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1508993024 bytes 2026-02-18T06:45:40.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1533913088 bytes 2026-02-18T06:45:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1581031424 bytes 2026-02-18T06:45:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1613621248 bytes 2026-02-18T06:45:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1660965888 bytes 2026-02-18T06:45:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1692516352 bytes 2026-02-18T06:45:40.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1713842176 bytes 2026-02-18T06:45:40.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1756447744 bytes 2026-02-18T06:45:40.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1808319488 bytes 2026-02-18T06:45:40.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1881540608 bytes 2026-02-18T06:45:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1911692288 bytes 2026-02-18T06:45:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1951409152 bytes 2026-02-18T06:45:40.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1952020480 bytes 2026-02-18T06:45:40.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2035545088 bytes 2026-02-18T06:45:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2063268864 bytes 2026-02-18T06:45:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2086152192 bytes 2026-02-18T06:45:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2138481664 bytes 2026-02-18T06:45:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2162966528 bytes 2026-02-18T06:45:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2214074368 bytes 2026-02-18T06:45:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2299908096 bytes 2026-02-18T06:45:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2318453760 bytes 2026-02-18T06:45:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2377178112 bytes 2026-02-18T06:45:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2405340160 bytes 2026-02-18T06:45:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2447590400 bytes 2026-02-18T06:45:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2488656896 bytes 2026-02-18T06:45:41.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2531804160 bytes 2026-02-18T06:45:41.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2609756160 bytes 2026-02-18T06:45:41.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2645058560 bytes 2026-02-18T06:45:41.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2728394752 bytes 2026-02-18T06:45:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2805334016 bytes 2026-02-18T06:45:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2828815360 bytes 2026-02-18T06:45:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2845348864 bytes 2026-02-18T06:45:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2896311296 bytes 2026-02-18T06:45:42.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2921895936 bytes 2026-02-18T06:45:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2964898816 bytes 2026-02-18T06:45:42.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2976965632 bytes 2026-02-18T06:45:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3077218304 bytes 2026-02-18T06:45:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3118781440 bytes 2026-02-18T06:45:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3164699648 bytes 2026-02-18T06:45:42.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3249266688 bytes 2026-02-18T06:45:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3377565696 bytes 2026-02-18T06:45:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3410770944 bytes 2026-02-18T06:45:42.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3449228288 bytes 2026-02-18T06:45:42.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3449839616 bytes 2026-02-18T06:45:42.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3496757248 bytes 2026-02-18T06:45:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3521750016 bytes 2026-02-18T06:45:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3563888640 bytes 2026-02-18T06:45:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3583369216 bytes 2026-02-18T06:45:43.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3622823936 bytes 2026-02-18T06:45:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3679260672 bytes 2026-02-18T06:45:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3738291200 bytes 2026-02-18T06:45:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3762791424 bytes 2026-02-18T06:45:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3775335424 bytes 2026-02-18T06:45:43.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3801952256 bytes 2026-02-18T06:45:43.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3836993536 bytes 2026-02-18T06:45:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3847760896 bytes 2026-02-18T06:45:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3872985088 bytes 2026-02-18T06:45:43.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3922713600 bytes 2026-02-18T06:45:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3960747008 bytes 2026-02-18T06:45:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3984207872 bytes 2026-02-18T06:45:43.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4030302208 bytes 2026-02-18T06:45:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4076456960 bytes 2026-02-18T06:45:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4104255488 bytes 2026-02-18T06:45:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4150039552 bytes 2026-02-18T06:45:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4176058368 bytes 2026-02-18T06:45:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4257096704 bytes 2026-02-18T06:45:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4281481216 bytes 2026-02-18T06:45:44.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4359313408 bytes 2026-02-18T06:45:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4410392576 bytes 2026-02-18T06:45:44.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4487278592 bytes 2026-02-18T06:45:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4524324864 bytes 2026-02-18T06:45:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4569030656 bytes 2026-02-18T06:45:44.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4623548416 bytes 2026-02-18T06:45:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4631684096 bytes 2026-02-18T06:45:44.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4652483584 bytes 2026-02-18T06:45:44.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4738770944 bytes 2026-02-18T06:45:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4770412544 bytes 2026-02-18T06:45:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4824845312 bytes 2026-02-18T06:45:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4892891136 bytes 2026-02-18T06:45:44.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4893530112 bytes 2026-02-18T06:45:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4932478976 bytes 2026-02-18T06:45:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4977455104 bytes 2026-02-18T06:45:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4983411712 bytes 2026-02-18T06:45:44.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5082853376 bytes 2026-02-18T06:45:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5139897344 bytes 2026-02-18T06:45:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5160964096 bytes 2026-02-18T06:45:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5189355520 bytes 2026-02-18T06:45:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5220868096 bytes 2026-02-18T06:45:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5221479424 bytes 2026-02-18T06:45:45.244 INFO:tasks.ceph.cluster2.osd.1.trial051.stderr:2026-02-18T06:45:45.241+0000 7fdadefcd640 -1 reset not still connected to 0x55feadf14270 2026-02-18T06:45:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5262205952 bytes 2026-02-18T06:45:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5301100544 bytes 2026-02-18T06:45:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5333784576 bytes 2026-02-18T06:45:45.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5375240192 bytes 2026-02-18T06:45:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5399968768 bytes 2026-02-18T06:45:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5440996352 bytes 2026-02-18T06:45:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5469313024 bytes 2026-02-18T06:45:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5513053184 bytes 2026-02-18T06:45:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5539633152 bytes 2026-02-18T06:45:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5547357184 bytes 2026-02-18T06:45:45.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5584633856 bytes 2026-02-18T06:45:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5653153792 bytes 2026-02-18T06:45:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5698092032 bytes 2026-02-18T06:45:45.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5749008384 bytes 2026-02-18T06:45:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5781028864 bytes 2026-02-18T06:45:46.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5786632192 bytes 2026-02-18T06:45:46.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5821462528 bytes 2026-02-18T06:45:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5851576320 bytes 2026-02-18T06:45:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5872505856 bytes 2026-02-18T06:45:46.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5962247168 bytes 2026-02-18T06:45:46.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5975463936 bytes 2026-02-18T06:45:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6055992320 bytes 2026-02-18T06:45:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6085404672 bytes 2026-02-18T06:45:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6133269504 bytes 2026-02-18T06:45:46.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6152363008 bytes 2026-02-18T06:45:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6189934592 bytes 2026-02-18T06:45:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6253227008 bytes 2026-02-18T06:45:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6269927424 bytes 2026-02-18T06:45:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6338015232 bytes 2026-02-18T06:45:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6359331840 bytes 2026-02-18T06:45:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6412985344 bytes 2026-02-18T06:45:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6413599744 bytes 2026-02-18T06:45:47.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6446743552 bytes 2026-02-18T06:45:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6480740352 bytes 2026-02-18T06:45:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6528746496 bytes 2026-02-18T06:45:47.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6557087744 bytes 2026-02-18T06:45:47.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6557701120 bytes 2026-02-18T06:45:47.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6633683968 bytes 2026-02-18T06:45:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6691380224 bytes 2026-02-18T06:45:47.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6773421056 bytes 2026-02-18T06:45:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6825362432 bytes 2026-02-18T06:45:47.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6836139008 bytes 2026-02-18T06:45:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6965692416 bytes 2026-02-18T06:45:47.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6973878272 bytes 2026-02-18T06:45:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7050554368 bytes 2026-02-18T06:45:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7084604416 bytes 2026-02-18T06:45:48.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7191465984 bytes 2026-02-18T06:45:48.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7239776256 bytes 2026-02-18T06:45:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7314518016 bytes 2026-02-18T06:45:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7370047488 bytes 2026-02-18T06:45:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7425673216 bytes 2026-02-18T06:45:48.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7460936704 bytes 2026-02-18T06:45:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7483897856 bytes 2026-02-18T06:45:48.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7524410368 bytes 2026-02-18T06:45:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7543264256 bytes 2026-02-18T06:45:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7606306816 bytes 2026-02-18T06:45:48.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7657828352 bytes 2026-02-18T06:45:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7766213632 bytes 2026-02-18T06:45:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7815925760 bytes 2026-02-18T06:45:48.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7853381632 bytes 2026-02-18T06:45:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7887832064 bytes 2026-02-18T06:45:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7953074176 bytes 2026-02-18T06:45:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7972226048 bytes 2026-02-18T06:45:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8011676672 bytes 2026-02-18T06:45:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8030576640 bytes 2026-02-18T06:45:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8097883136 bytes 2026-02-18T06:45:49.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8106112000 bytes 2026-02-18T06:45:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8147160064 bytes 2026-02-18T06:45:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8175899648 bytes 2026-02-18T06:45:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8275633152 bytes 2026-02-18T06:45:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8337708032 bytes 2026-02-18T06:45:49.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8386348032 bytes 2026-02-18T06:45:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8399048704 bytes 2026-02-18T06:45:49.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8451517440 bytes 2026-02-18T06:45:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8494049280 bytes 2026-02-18T06:45:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8546371584 bytes 2026-02-18T06:45:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8590055424 bytes 2026-02-18T06:45:49.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8612967424 bytes 2026-02-18T06:45:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8683079680 bytes 2026-02-18T06:45:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8700530688 bytes 2026-02-18T06:45:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8702514176 bytes 2026-02-18T06:45:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8756028416 bytes 2026-02-18T06:45:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8821210112 bytes 2026-02-18T06:45:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8849865728 bytes 2026-02-18T06:45:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8886855680 bytes 2026-02-18T06:45:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8915867648 bytes 2026-02-18T06:45:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8989972480 bytes 2026-02-18T06:45:50.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 9036539904 bytes 2026-02-18T06:45:50.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 9037136896 bytes 2026-02-18T06:45:50.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 9090539520 bytes 2026-02-18T06:45:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 9124610048 bytes 2026-02-18T06:45:50.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 9144342528 bytes 2026-02-18T06:45:50.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 9223777280 bytes 2026-02-18T06:45:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 9293633536 bytes 2026-02-18T06:45:50.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 9376713728 bytes 2026-02-18T06:45:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 9443591168 bytes 2026-02-18T06:45:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 9468704768 bytes 2026-02-18T06:45:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 9548735488 bytes 2026-02-18T06:45:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 9573592064 bytes 2026-02-18T06:45:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 9574203392 bytes 2026-02-18T06:45:51.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 9653159936 bytes 2026-02-18T06:45:51.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 9687429120 bytes 2026-02-18T06:45:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 9708602368 bytes 2026-02-18T06:45:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23000 IOs, 9709203456 bytes 2026-02-18T06:45:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23100 IOs, 9721964544 bytes 2026-02-18T06:45:51.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23200 IOs, 9764393984 bytes 2026-02-18T06:45:51.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23300 IOs, 9832597504 bytes 2026-02-18T06:45:51.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23400 IOs, 9864221696 bytes 2026-02-18T06:45:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23500 IOs, 9896697856 bytes 2026-02-18T06:45:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23600 IOs, 9927769088 bytes 2026-02-18T06:45:51.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23700 IOs, 9950568448 bytes 2026-02-18T06:45:51.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23800 IOs, 10022445056 bytes 2026-02-18T06:45:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23900 IOs, 10112650240 bytes 2026-02-18T06:45:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24000 IOs, 10151654400 bytes 2026-02-18T06:45:52.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:24100 IOs, 10269142016 bytes 2026-02-18T06:45:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_2 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:45:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:45:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:45:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:45:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:45:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_2 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2026-02-18T06:45:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:45:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:45:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-18T06:45:52.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:45:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_2': 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 128623c9eca0 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.128623c9eca0 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:45:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:45:52 2026 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:45:52 2026 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:45:52 2026 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e6c1e58d-7558-4bf6-9061-94218ae350a1 2026-02-18T06:45:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:45:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:45:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-18T06:45:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:45:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:45:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:45:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:45:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T06:45:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:45:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:45:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 89911296 bytes 2026-02-18T06:45:53.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 96930816 bytes 2026-02-18T06:45:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 115275776 bytes 2026-02-18T06:45:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 137122816 bytes 2026-02-18T06:45:53.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 141135872 bytes 2026-02-18T06:45:53.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 208309248 bytes 2026-02-18T06:45:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 274370560 bytes 2026-02-18T06:45:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 337149952 bytes 2026-02-18T06:45:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 388605952 bytes 2026-02-18T06:45:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 434444288 bytes 2026-02-18T06:45:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 480363520 bytes 2026-02-18T06:45:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 498720768 bytes 2026-02-18T06:45:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 540355584 bytes 2026-02-18T06:45:53.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 579258368 bytes 2026-02-18T06:45:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 669116416 bytes 2026-02-18T06:45:53.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 708897792 bytes 2026-02-18T06:45:53.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 741780480 bytes 2026-02-18T06:45:54.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 797810688 bytes 2026-02-18T06:45:54.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 823746560 bytes 2026-02-18T06:45:54.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 918646784 bytes 2026-02-18T06:45:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 970345472 bytes 2026-02-18T06:45:54.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 988393472 bytes 2026-02-18T06:45:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1011609600 bytes 2026-02-18T06:45:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1107909632 bytes 2026-02-18T06:45:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1204504576 bytes 2026-02-18T06:45:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1245421568 bytes 2026-02-18T06:45:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1265648640 bytes 2026-02-18T06:45:54.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1338804224 bytes 2026-02-18T06:45:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1358971904 bytes 2026-02-18T06:45:54.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1365966848 bytes 2026-02-18T06:45:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1386505216 bytes 2026-02-18T06:45:54.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1424528384 bytes 2026-02-18T06:45:54.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1425136640 bytes 2026-02-18T06:45:54.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1464011776 bytes 2026-02-18T06:45:54.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1488691200 bytes 2026-02-18T06:45:55.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1536783360 bytes 2026-02-18T06:45:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1583944704 bytes 2026-02-18T06:45:55.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1614590976 bytes 2026-02-18T06:45:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1661850624 bytes 2026-02-18T06:45:55.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1687483392 bytes 2026-02-18T06:45:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1688118272 bytes 2026-02-18T06:45:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1707165696 bytes 2026-02-18T06:45:55.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1793486848 bytes 2026-02-18T06:45:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1850923008 bytes 2026-02-18T06:45:55.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1886002176 bytes 2026-02-18T06:45:55.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1893458944 bytes 2026-02-18T06:45:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1973932032 bytes 2026-02-18T06:45:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2048589824 bytes 2026-02-18T06:45:55.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2130712576 bytes 2026-02-18T06:45:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2144171008 bytes 2026-02-18T06:45:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2176662528 bytes 2026-02-18T06:45:55.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2229630976 bytes 2026-02-18T06:45:55.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2261963776 bytes 2026-02-18T06:45:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2279971840 bytes 2026-02-18T06:45:55.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2280562688 bytes 2026-02-18T06:45:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2322666496 bytes 2026-02-18T06:45:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2323278848 bytes 2026-02-18T06:45:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2364459008 bytes 2026-02-18T06:45:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2403674112 bytes 2026-02-18T06:45:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2435976192 bytes 2026-02-18T06:45:56.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2456160256 bytes 2026-02-18T06:45:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2501221376 bytes 2026-02-18T06:45:56.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2555789312 bytes 2026-02-18T06:45:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2556413952 bytes 2026-02-18T06:45:56.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2611698688 bytes 2026-02-18T06:45:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2640555008 bytes 2026-02-18T06:45:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2641195008 bytes 2026-02-18T06:45:56.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2644832256 bytes 2026-02-18T06:45:56.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2655782912 bytes 2026-02-18T06:45:56.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2690384896 bytes 2026-02-18T06:45:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2745009152 bytes 2026-02-18T06:45:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2782283776 bytes 2026-02-18T06:45:56.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2834903040 bytes 2026-02-18T06:45:56.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2877245440 bytes 2026-02-18T06:45:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2897019904 bytes 2026-02-18T06:45:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2966775808 bytes 2026-02-18T06:45:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3021182976 bytes 2026-02-18T06:45:57.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3048152064 bytes 2026-02-18T06:45:57.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3083984896 bytes 2026-02-18T06:45:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3090731008 bytes 2026-02-18T06:45:57.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3103257600 bytes 2026-02-18T06:45:57.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3119772672 bytes 2026-02-18T06:45:57.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3171129344 bytes 2026-02-18T06:45:57.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3240407040 bytes 2026-02-18T06:45:57.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3311856640 bytes 2026-02-18T06:45:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3329986560 bytes 2026-02-18T06:45:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3388498944 bytes 2026-02-18T06:45:57.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3419284480 bytes 2026-02-18T06:45:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3508719616 bytes 2026-02-18T06:45:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3534737408 bytes 2026-02-18T06:45:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3571259392 bytes 2026-02-18T06:45:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3620358144 bytes 2026-02-18T06:45:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3695423488 bytes 2026-02-18T06:45:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3696033792 bytes 2026-02-18T06:45:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3718668288 bytes 2026-02-18T06:45:58.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3756528640 bytes 2026-02-18T06:45:58.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3786290176 bytes 2026-02-18T06:45:58.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3800553472 bytes 2026-02-18T06:45:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3808230400 bytes 2026-02-18T06:45:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3890945024 bytes 2026-02-18T06:45:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3914016768 bytes 2026-02-18T06:45:58.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4001497088 bytes 2026-02-18T06:45:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4131728384 bytes 2026-02-18T06:45:58.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4147734528 bytes 2026-02-18T06:45:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4210246656 bytes 2026-02-18T06:45:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4240532480 bytes 2026-02-18T06:45:58.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4292983808 bytes 2026-02-18T06:45:58.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4410441728 bytes 2026-02-18T06:45:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4464532480 bytes 2026-02-18T06:45:58.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4480798720 bytes 2026-02-18T06:45:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4525489152 bytes 2026-02-18T06:45:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4566040576 bytes 2026-02-18T06:45:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4593251328 bytes 2026-02-18T06:45:59.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4639744000 bytes 2026-02-18T06:45:59.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4640356352 bytes 2026-02-18T06:45:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4698651648 bytes 2026-02-18T06:45:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4752544768 bytes 2026-02-18T06:45:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4804217856 bytes 2026-02-18T06:45:59.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4842380288 bytes 2026-02-18T06:45:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4846005248 bytes 2026-02-18T06:45:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4865561600 bytes 2026-02-18T06:45:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4906052608 bytes 2026-02-18T06:45:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4966877184 bytes 2026-02-18T06:45:59.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4995872768 bytes 2026-02-18T06:45:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5025519616 bytes 2026-02-18T06:45:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5089349632 bytes 2026-02-18T06:45:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5094751232 bytes 2026-02-18T06:45:59.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5116485632 bytes 2026-02-18T06:45:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5148256256 bytes 2026-02-18T06:45:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5167332352 bytes 2026-02-18T06:45:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5184992256 bytes 2026-02-18T06:46:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5205468160 bytes 2026-02-18T06:46:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5230350336 bytes 2026-02-18T06:46:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5296036864 bytes 2026-02-18T06:46:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5329387520 bytes 2026-02-18T06:46:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5337931776 bytes 2026-02-18T06:46:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5377132544 bytes 2026-02-18T06:46:00.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5436761088 bytes 2026-02-18T06:46:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5437384704 bytes 2026-02-18T06:46:00.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5459866624 bytes 2026-02-18T06:46:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5545526272 bytes 2026-02-18T06:46:00.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5600195584 bytes 2026-02-18T06:46:00.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5654631424 bytes 2026-02-18T06:46:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5655245824 bytes 2026-02-18T06:46:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5684554752 bytes 2026-02-18T06:46:00.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5685186560 bytes 2026-02-18T06:46:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5750749184 bytes 2026-02-18T06:46:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5762574336 bytes 2026-02-18T06:46:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5838768128 bytes 2026-02-18T06:46:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 5886127104 bytes 2026-02-18T06:46:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 5912598528 bytes 2026-02-18T06:46:01.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 5953723392 bytes 2026-02-18T06:46:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6062950400 bytes 2026-02-18T06:46:01.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6085797888 bytes 2026-02-18T06:46:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6122011648 bytes 2026-02-18T06:46:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6147313664 bytes 2026-02-18T06:46:01.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6210565120 bytes 2026-02-18T06:46:01.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6251610112 bytes 2026-02-18T06:46:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6274755584 bytes 2026-02-18T06:46:01.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6290130944 bytes 2026-02-18T06:46:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6340409344 bytes 2026-02-18T06:46:01.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6375754752 bytes 2026-02-18T06:46:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6419802112 bytes 2026-02-18T06:46:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6478759936 bytes 2026-02-18T06:46:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6524717056 bytes 2026-02-18T06:46:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6557014016 bytes 2026-02-18T06:46:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6600994816 bytes 2026-02-18T06:46:02.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6677232640 bytes 2026-02-18T06:46:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6738714624 bytes 2026-02-18T06:46:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6812988416 bytes 2026-02-18T06:46:02.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6823496704 bytes 2026-02-18T06:46:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6878000128 bytes 2026-02-18T06:46:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6893795328 bytes 2026-02-18T06:46:02.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6961091584 bytes 2026-02-18T06:46:02.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7038112768 bytes 2026-02-18T06:46:02.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7123044352 bytes 2026-02-18T06:46:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7149885440 bytes 2026-02-18T06:46:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7203294208 bytes 2026-02-18T06:46:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7286351872 bytes 2026-02-18T06:46:03.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7290870784 bytes 2026-02-18T06:46:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7327212544 bytes 2026-02-18T06:46:03.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7388023808 bytes 2026-02-18T06:46:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7425362944 bytes 2026-02-18T06:46:03.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7483886592 bytes 2026-02-18T06:46:03.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7555419136 bytes 2026-02-18T06:46:03.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7570135040 bytes 2026-02-18T06:46:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7591196672 bytes 2026-02-18T06:46:03.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7683538944 bytes 2026-02-18T06:46:03.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7710698496 bytes 2026-02-18T06:46:03.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7741940736 bytes 2026-02-18T06:46:03.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7776437248 bytes 2026-02-18T06:46:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7846610944 bytes 2026-02-18T06:46:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7864124416 bytes 2026-02-18T06:46:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7875749888 bytes 2026-02-18T06:46:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7890927616 bytes 2026-02-18T06:46:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7939906560 bytes 2026-02-18T06:46:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7986204672 bytes 2026-02-18T06:46:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8033668096 bytes 2026-02-18T06:46:04.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8083578880 bytes 2026-02-18T06:46:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8142377984 bytes 2026-02-18T06:46:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8147885056 bytes 2026-02-18T06:46:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8219723776 bytes 2026-02-18T06:46:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8242446336 bytes 2026-02-18T06:46:04.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8296837120 bytes 2026-02-18T06:46:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8383289344 bytes 2026-02-18T06:46:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8416947200 bytes 2026-02-18T06:46:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8419606528 bytes 2026-02-18T06:46:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8502838272 bytes 2026-02-18T06:46:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8538693632 bytes 2026-02-18T06:46:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8578594816 bytes 2026-02-18T06:46:04.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8640533504 bytes 2026-02-18T06:46:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8663688192 bytes 2026-02-18T06:46:04.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8705290240 bytes 2026-02-18T06:46:04.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 8733707264 bytes 2026-02-18T06:46:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 8751668224 bytes 2026-02-18T06:46:05.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:46:05.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_3 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:46:05.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:46:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:46:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:46:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_3 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-18T06:46:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:46:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-18T06:46:06.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_3': 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1292855509c8 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1292855509c8 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:46:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:05 2026 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:05 2026 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:05 2026 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: ef7fee7b-f8b3-4ef1-ade2-f49cb926b8d3 2026-02-18T06:46:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:46:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:46:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:46:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:46:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:46:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 22806528 bytes 2026-02-18T06:46:06.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 23407616 bytes 2026-02-18T06:46:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 84692992 bytes 2026-02-18T06:46:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 177012736 bytes 2026-02-18T06:46:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 181921792 bytes 2026-02-18T06:46:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 234750976 bytes 2026-02-18T06:46:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 235360256 bytes 2026-02-18T06:46:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 268808192 bytes 2026-02-18T06:46:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 302418944 bytes 2026-02-18T06:46:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 354988032 bytes 2026-02-18T06:46:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 361957376 bytes 2026-02-18T06:46:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 394179584 bytes 2026-02-18T06:46:06.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 395069440 bytes 2026-02-18T06:46:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 426529792 bytes 2026-02-18T06:46:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 489098240 bytes 2026-02-18T06:46:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 533753856 bytes 2026-02-18T06:46:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 629916672 bytes 2026-02-18T06:46:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 668322816 bytes 2026-02-18T06:46:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 710563840 bytes 2026-02-18T06:46:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 745808896 bytes 2026-02-18T06:46:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 764406784 bytes 2026-02-18T06:46:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 777961472 bytes 2026-02-18T06:46:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 843376640 bytes 2026-02-18T06:46:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 888264704 bytes 2026-02-18T06:46:07.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 943911936 bytes 2026-02-18T06:46:07.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 950646784 bytes 2026-02-18T06:46:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 956227584 bytes 2026-02-18T06:46:07.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1054039040 bytes 2026-02-18T06:46:07.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1056804864 bytes 2026-02-18T06:46:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1123214336 bytes 2026-02-18T06:46:07.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1141661696 bytes 2026-02-18T06:46:07.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1184315392 bytes 2026-02-18T06:46:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1202226176 bytes 2026-02-18T06:46:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1222429696 bytes 2026-02-18T06:46:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1272838144 bytes 2026-02-18T06:46:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1364272128 bytes 2026-02-18T06:46:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1399434240 bytes 2026-02-18T06:46:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1449690112 bytes 2026-02-18T06:46:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1500498944 bytes 2026-02-18T06:46:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1513792512 bytes 2026-02-18T06:46:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1630849024 bytes 2026-02-18T06:46:08.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1656624128 bytes 2026-02-18T06:46:08.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1659608064 bytes 2026-02-18T06:46:08.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1743185920 bytes 2026-02-18T06:46:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1811985408 bytes 2026-02-18T06:46:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1866531840 bytes 2026-02-18T06:46:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1904619520 bytes 2026-02-18T06:46:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2003509248 bytes 2026-02-18T06:46:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2004848640 bytes 2026-02-18T06:46:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2017930240 bytes 2026-02-18T06:46:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2040525824 bytes 2026-02-18T06:46:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2066954240 bytes 2026-02-18T06:46:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2090702848 bytes 2026-02-18T06:46:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2118225920 bytes 2026-02-18T06:46:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2170977280 bytes 2026-02-18T06:46:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2225100800 bytes 2026-02-18T06:46:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2235277312 bytes 2026-02-18T06:46:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2275010560 bytes 2026-02-18T06:46:09.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2358043648 bytes 2026-02-18T06:46:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2399214592 bytes 2026-02-18T06:46:09.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2465758208 bytes 2026-02-18T06:46:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2534141952 bytes 2026-02-18T06:46:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2554084352 bytes 2026-02-18T06:46:09.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2626640896 bytes 2026-02-18T06:46:09.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2645702656 bytes 2026-02-18T06:46:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2674991104 bytes 2026-02-18T06:46:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2685844480 bytes 2026-02-18T06:46:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2755505152 bytes 2026-02-18T06:46:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2824320000 bytes 2026-02-18T06:46:10.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2824946688 bytes 2026-02-18T06:46:10.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2925867008 bytes 2026-02-18T06:46:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2979753984 bytes 2026-02-18T06:46:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3038746624 bytes 2026-02-18T06:46:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3135519744 bytes 2026-02-18T06:46:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3136172032 bytes 2026-02-18T06:46:10.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3148264448 bytes 2026-02-18T06:46:10.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3185930240 bytes 2026-02-18T06:46:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3217787904 bytes 2026-02-18T06:46:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3280361472 bytes 2026-02-18T06:46:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3306251264 bytes 2026-02-18T06:46:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3395891200 bytes 2026-02-18T06:46:10.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3423629312 bytes 2026-02-18T06:46:10.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3506216960 bytes 2026-02-18T06:46:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3525475328 bytes 2026-02-18T06:46:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3582716928 bytes 2026-02-18T06:46:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3652727808 bytes 2026-02-18T06:46:11.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3715557376 bytes 2026-02-18T06:46:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3770371072 bytes 2026-02-18T06:46:11.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3812092928 bytes 2026-02-18T06:46:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3844241408 bytes 2026-02-18T06:46:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3849120768 bytes 2026-02-18T06:46:11.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3873750016 bytes 2026-02-18T06:46:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3916175360 bytes 2026-02-18T06:46:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3959902208 bytes 2026-02-18T06:46:11.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3969220608 bytes 2026-02-18T06:46:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4003850240 bytes 2026-02-18T06:46:11.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4006660096 bytes 2026-02-18T06:46:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4037289984 bytes 2026-02-18T06:46:11.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4053952512 bytes 2026-02-18T06:46:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4129632256 bytes 2026-02-18T06:46:11.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4207235072 bytes 2026-02-18T06:46:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4285284352 bytes 2026-02-18T06:46:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4337786880 bytes 2026-02-18T06:46:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4413483008 bytes 2026-02-18T06:46:12.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4453011456 bytes 2026-02-18T06:46:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4487964672 bytes 2026-02-18T06:46:12.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4516347904 bytes 2026-02-18T06:46:12.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4531892224 bytes 2026-02-18T06:46:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4634002432 bytes 2026-02-18T06:46:12.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4693830656 bytes 2026-02-18T06:46:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4753951744 bytes 2026-02-18T06:46:12.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4839550976 bytes 2026-02-18T06:46:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4860862464 bytes 2026-02-18T06:46:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4930342912 bytes 2026-02-18T06:46:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5000820736 bytes 2026-02-18T06:46:13.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5004821504 bytes 2026-02-18T06:46:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5028478976 bytes 2026-02-18T06:46:13.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5051587584 bytes 2026-02-18T06:46:13.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5061424128 bytes 2026-02-18T06:46:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5099514880 bytes 2026-02-18T06:46:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5140583424 bytes 2026-02-18T06:46:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5176283136 bytes 2026-02-18T06:46:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5238375424 bytes 2026-02-18T06:46:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5286363136 bytes 2026-02-18T06:46:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5322072064 bytes 2026-02-18T06:46:13.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5378084864 bytes 2026-02-18T06:46:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5441371136 bytes 2026-02-18T06:46:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5486577664 bytes 2026-02-18T06:46:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5487183872 bytes 2026-02-18T06:46:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5521864704 bytes 2026-02-18T06:46:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5525048320 bytes 2026-02-18T06:46:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5563641856 bytes 2026-02-18T06:46:13.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5619644416 bytes 2026-02-18T06:46:14.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5661167616 bytes 2026-02-18T06:46:14.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5749647360 bytes 2026-02-18T06:46:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5848114176 bytes 2026-02-18T06:46:14.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5864597504 bytes 2026-02-18T06:46:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5895008256 bytes 2026-02-18T06:46:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5935761408 bytes 2026-02-18T06:46:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5991608320 bytes 2026-02-18T06:46:14.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5992228864 bytes 2026-02-18T06:46:14.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6042147840 bytes 2026-02-18T06:46:14.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6078478336 bytes 2026-02-18T06:46:14.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6126016512 bytes 2026-02-18T06:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6156301312 bytes 2026-02-18T06:46:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6200185856 bytes 2026-02-18T06:46:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6260948992 bytes 2026-02-18T06:46:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6261569536 bytes 2026-02-18T06:46:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6262184960 bytes 2026-02-18T06:46:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6296570880 bytes 2026-02-18T06:46:14.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6353139712 bytes 2026-02-18T06:46:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6392009728 bytes 2026-02-18T06:46:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6425133056 bytes 2026-02-18T06:46:15.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6507287552 bytes 2026-02-18T06:46:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6581031936 bytes 2026-02-18T06:46:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6625210368 bytes 2026-02-18T06:46:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6682130432 bytes 2026-02-18T06:46:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6713088000 bytes 2026-02-18T06:46:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6713705472 bytes 2026-02-18T06:46:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6749492224 bytes 2026-02-18T06:46:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6820800512 bytes 2026-02-18T06:46:15.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6859018240 bytes 2026-02-18T06:46:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6939244544 bytes 2026-02-18T06:46:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6971850752 bytes 2026-02-18T06:46:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7020879872 bytes 2026-02-18T06:46:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7043753984 bytes 2026-02-18T06:46:15.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7105374208 bytes 2026-02-18T06:46:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7130101760 bytes 2026-02-18T06:46:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7199889408 bytes 2026-02-18T06:46:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7248381952 bytes 2026-02-18T06:46:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7293693952 bytes 2026-02-18T06:46:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7348403200 bytes 2026-02-18T06:46:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7380803584 bytes 2026-02-18T06:46:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7412115456 bytes 2026-02-18T06:46:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7443468288 bytes 2026-02-18T06:46:16.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7485228032 bytes 2026-02-18T06:46:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7485837312 bytes 2026-02-18T06:46:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7523121152 bytes 2026-02-18T06:46:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7551798272 bytes 2026-02-18T06:46:16.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7615562752 bytes 2026-02-18T06:46:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7640572928 bytes 2026-02-18T06:46:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7669739520 bytes 2026-02-18T06:46:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7728324608 bytes 2026-02-18T06:46:16.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7790429184 bytes 2026-02-18T06:46:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7875434496 bytes 2026-02-18T06:46:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:46:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_4 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:46:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:46:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:46:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_4 2026-02-18T06:46:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:46:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-18T06:46:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:46:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_4': 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 129e826f86de 2026-02-18T06:46:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.129e826f86de 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:17 2026 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:17 2026 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:17 2026 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:46:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e7796cb1-112a-4c40-81fa-5cc2bae63d79 2026-02-18T06:46:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:46:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:46:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-18T06:46:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:46:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:46:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:46:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:46:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:46:18.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 84449280 bytes 2026-02-18T06:46:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 161553408 bytes 2026-02-18T06:46:18.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 256351232 bytes 2026-02-18T06:46:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 283505664 bytes 2026-02-18T06:46:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 309909504 bytes 2026-02-18T06:46:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 409806848 bytes 2026-02-18T06:46:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 456119296 bytes 2026-02-18T06:46:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 473351168 bytes 2026-02-18T06:46:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 539276288 bytes 2026-02-18T06:46:18.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 604891136 bytes 2026-02-18T06:46:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 624748544 bytes 2026-02-18T06:46:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 668891136 bytes 2026-02-18T06:46:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 779357184 bytes 2026-02-18T06:46:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 802274304 bytes 2026-02-18T06:46:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 875961344 bytes 2026-02-18T06:46:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 926788608 bytes 2026-02-18T06:46:19.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 985372672 bytes 2026-02-18T06:46:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 1039689728 bytes 2026-02-18T06:46:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 1040318464 bytes 2026-02-18T06:46:19.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1151164416 bytes 2026-02-18T06:46:19.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1180972032 bytes 2026-02-18T06:46:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1247275008 bytes 2026-02-18T06:46:19.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1285885952 bytes 2026-02-18T06:46:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1328104448 bytes 2026-02-18T06:46:19.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1432467456 bytes 2026-02-18T06:46:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1465580544 bytes 2026-02-18T06:46:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1471474688 bytes 2026-02-18T06:46:19.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1501844480 bytes 2026-02-18T06:46:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1534425088 bytes 2026-02-18T06:46:20.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1585387520 bytes 2026-02-18T06:46:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1609620480 bytes 2026-02-18T06:46:20.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1659933696 bytes 2026-02-18T06:46:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1709573120 bytes 2026-02-18T06:46:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1736528896 bytes 2026-02-18T06:46:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1779155968 bytes 2026-02-18T06:46:20.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1806128128 bytes 2026-02-18T06:46:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1843481600 bytes 2026-02-18T06:46:20.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1921095680 bytes 2026-02-18T06:46:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1975022592 bytes 2026-02-18T06:46:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2032089088 bytes 2026-02-18T06:46:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2052584448 bytes 2026-02-18T06:46:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2080752640 bytes 2026-02-18T06:46:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2091149312 bytes 2026-02-18T06:46:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2110484480 bytes 2026-02-18T06:46:20.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2130458624 bytes 2026-02-18T06:46:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2157644800 bytes 2026-02-18T06:46:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2248695808 bytes 2026-02-18T06:46:21.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2305167360 bytes 2026-02-18T06:46:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2309734400 bytes 2026-02-18T06:46:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2321634304 bytes 2026-02-18T06:46:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2375475200 bytes 2026-02-18T06:46:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2387185664 bytes 2026-02-18T06:46:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2417883136 bytes 2026-02-18T06:46:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2449063936 bytes 2026-02-18T06:46:21.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2458199040 bytes 2026-02-18T06:46:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2475207680 bytes 2026-02-18T06:46:21.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2514247680 bytes 2026-02-18T06:46:21.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2591443968 bytes 2026-02-18T06:46:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2672125952 bytes 2026-02-18T06:46:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2689507328 bytes 2026-02-18T06:46:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2766102528 bytes 2026-02-18T06:46:21.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2849886208 bytes 2026-02-18T06:46:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2850518016 bytes 2026-02-18T06:46:21.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2879622144 bytes 2026-02-18T06:46:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2907423744 bytes 2026-02-18T06:46:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2936856576 bytes 2026-02-18T06:46:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2973726720 bytes 2026-02-18T06:46:22.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3015650304 bytes 2026-02-18T06:46:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3082589184 bytes 2026-02-18T06:46:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3127910400 bytes 2026-02-18T06:46:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3175729152 bytes 2026-02-18T06:46:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3200260096 bytes 2026-02-18T06:46:22.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3255520256 bytes 2026-02-18T06:46:22.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3294654464 bytes 2026-02-18T06:46:22.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3317917696 bytes 2026-02-18T06:46:22.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3383753728 bytes 2026-02-18T06:46:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3467495424 bytes 2026-02-18T06:46:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3515885568 bytes 2026-02-18T06:46:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3579880448 bytes 2026-02-18T06:46:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3669379072 bytes 2026-02-18T06:46:22.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3687933952 bytes 2026-02-18T06:46:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3732091904 bytes 2026-02-18T06:46:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3743761408 bytes 2026-02-18T06:46:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3823365120 bytes 2026-02-18T06:46:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3968184320 bytes 2026-02-18T06:46:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4023221248 bytes 2026-02-18T06:46:23.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4152860672 bytes 2026-02-18T06:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4213821440 bytes 2026-02-18T06:46:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4268745728 bytes 2026-02-18T06:46:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4293656576 bytes 2026-02-18T06:46:23.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4331009024 bytes 2026-02-18T06:46:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4374940672 bytes 2026-02-18T06:46:23.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4411130880 bytes 2026-02-18T06:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4473706496 bytes 2026-02-18T06:46:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4497595392 bytes 2026-02-18T06:46:23.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4505001984 bytes 2026-02-18T06:46:23.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4565985280 bytes 2026-02-18T06:46:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4566591488 bytes 2026-02-18T06:46:24.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4604510208 bytes 2026-02-18T06:46:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4653071360 bytes 2026-02-18T06:46:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4673582080 bytes 2026-02-18T06:46:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4743129088 bytes 2026-02-18T06:46:24.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4784977920 bytes 2026-02-18T06:46:24.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4827376640 bytes 2026-02-18T06:46:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4827957248 bytes 2026-02-18T06:46:24.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4903017472 bytes 2026-02-18T06:46:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4944804864 bytes 2026-02-18T06:46:24.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4981585920 bytes 2026-02-18T06:46:24.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4995980288 bytes 2026-02-18T06:46:24.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5063583744 bytes 2026-02-18T06:46:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5140318208 bytes 2026-02-18T06:46:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5215036416 bytes 2026-02-18T06:46:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5294485504 bytes 2026-02-18T06:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5309791232 bytes 2026-02-18T06:46:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5377184768 bytes 2026-02-18T06:46:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5400937472 bytes 2026-02-18T06:46:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5421678592 bytes 2026-02-18T06:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5463056384 bytes 2026-02-18T06:46:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5517227008 bytes 2026-02-18T06:46:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5539902464 bytes 2026-02-18T06:46:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5554094080 bytes 2026-02-18T06:46:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5562078208 bytes 2026-02-18T06:46:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5616980992 bytes 2026-02-18T06:46:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5632206848 bytes 2026-02-18T06:46:25.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5652629504 bytes 2026-02-18T06:46:25.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5677665280 bytes 2026-02-18T06:46:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5703349248 bytes 2026-02-18T06:46:25.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5721613312 bytes 2026-02-18T06:46:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5770784768 bytes 2026-02-18T06:46:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5855381504 bytes 2026-02-18T06:46:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5890410496 bytes 2026-02-18T06:46:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5940373504 bytes 2026-02-18T06:46:25.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6027125760 bytes 2026-02-18T06:46:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6041634816 bytes 2026-02-18T06:46:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6066466816 bytes 2026-02-18T06:46:26.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6107261952 bytes 2026-02-18T06:46:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6180631552 bytes 2026-02-18T06:46:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6197938176 bytes 2026-02-18T06:46:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6246739968 bytes 2026-02-18T06:46:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6283912192 bytes 2026-02-18T06:46:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6315091968 bytes 2026-02-18T06:46:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6315795456 bytes 2026-02-18T06:46:26.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6343626752 bytes 2026-02-18T06:46:26.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6377247744 bytes 2026-02-18T06:46:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6452776960 bytes 2026-02-18T06:46:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6453387264 bytes 2026-02-18T06:46:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6472835072 bytes 2026-02-18T06:46:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6547232768 bytes 2026-02-18T06:46:26.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6568949760 bytes 2026-02-18T06:46:26.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6569539584 bytes 2026-02-18T06:46:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6656619520 bytes 2026-02-18T06:46:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6732545024 bytes 2026-02-18T06:46:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6767100928 bytes 2026-02-18T06:46:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6779734016 bytes 2026-02-18T06:46:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6823688192 bytes 2026-02-18T06:46:27.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6915647488 bytes 2026-02-18T06:46:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7007385600 bytes 2026-02-18T06:46:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7074719744 bytes 2026-02-18T06:46:27.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7124115456 bytes 2026-02-18T06:46:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7243530240 bytes 2026-02-18T06:46:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7307100160 bytes 2026-02-18T06:46:27.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7338691584 bytes 2026-02-18T06:46:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7356235776 bytes 2026-02-18T06:46:27.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7356835840 bytes 2026-02-18T06:46:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7372797952 bytes 2026-02-18T06:46:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7392439296 bytes 2026-02-18T06:46:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7468869632 bytes 2026-02-18T06:46:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7471217664 bytes 2026-02-18T06:46:28.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7514487808 bytes 2026-02-18T06:46:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7539222528 bytes 2026-02-18T06:46:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7617560576 bytes 2026-02-18T06:46:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7632425984 bytes 2026-02-18T06:46:28.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7681754112 bytes 2026-02-18T06:46:28.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7743919104 bytes 2026-02-18T06:46:28.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7769796608 bytes 2026-02-18T06:46:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7827089408 bytes 2026-02-18T06:46:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7875200000 bytes 2026-02-18T06:46:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7880345600 bytes 2026-02-18T06:46:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7987564544 bytes 2026-02-18T06:46:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 8046391296 bytes 2026-02-18T06:46:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 8049781760 bytes 2026-02-18T06:46:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 8061811712 bytes 2026-02-18T06:46:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 8130333696 bytes 2026-02-18T06:46:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 8185020416 bytes 2026-02-18T06:46:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8278958080 bytes 2026-02-18T06:46:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8315895808 bytes 2026-02-18T06:46:29.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8376202240 bytes 2026-02-18T06:46:29.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8439205888 bytes 2026-02-18T06:46:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8493737984 bytes 2026-02-18T06:46:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8546326528 bytes 2026-02-18T06:46:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8651047936 bytes 2026-02-18T06:46:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8706350080 bytes 2026-02-18T06:46:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8727429120 bytes 2026-02-18T06:46:29.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8829416448 bytes 2026-02-18T06:46:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8864017408 bytes 2026-02-18T06:46:29.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8908461056 bytes 2026-02-18T06:46:30.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 9001393152 bytes 2026-02-18T06:46:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 9052453888 bytes 2026-02-18T06:46:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 9096152064 bytes 2026-02-18T06:46:30.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 9122033664 bytes 2026-02-18T06:46:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 9172311040 bytes 2026-02-18T06:46:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 9213140992 bytes 2026-02-18T06:46:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 9285460992 bytes 2026-02-18T06:46:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 9307406336 bytes 2026-02-18T06:46:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 9334774784 bytes 2026-02-18T06:46:30.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 9375753216 bytes 2026-02-18T06:46:30.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 9426462720 bytes 2026-02-18T06:46:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 9447641088 bytes 2026-02-18T06:46:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 9463865344 bytes 2026-02-18T06:46:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 9464488960 bytes 2026-02-18T06:46:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 9553449984 bytes 2026-02-18T06:46:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 9576791040 bytes 2026-02-18T06:46:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 9682316288 bytes 2026-02-18T06:46:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 9741379584 bytes 2026-02-18T06:46:31.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 9782467584 bytes 2026-02-18T06:46:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 9800569856 bytes 2026-02-18T06:46:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 9873665024 bytes 2026-02-18T06:46:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 9891927040 bytes 2026-02-18T06:46:31.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 9953787904 bytes 2026-02-18T06:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 9982693376 bytes 2026-02-18T06:46:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 10025341952 bytes 2026-02-18T06:46:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 10083232768 bytes 2026-02-18T06:46:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 10089725952 bytes 2026-02-18T06:46:31.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 10115585024 bytes 2026-02-18T06:46:31.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 10171370496 bytes 2026-02-18T06:46:31.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 10205730816 bytes 2026-02-18T06:46:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 10263054336 bytes 2026-02-18T06:46:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 10312013824 bytes 2026-02-18T06:46:32.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 10312615936 bytes 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_5 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_5 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2026-02-18T06:46:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:46:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-18T06:46:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:46:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_5': 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12aa5d242bca 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12aa5d242bca 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:46:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:32 2026 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:32 2026 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:32 2026 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: db0ba49e-c26a-4f8a-bc0c-a15701fe2493 2026-02-18T06:46:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:46:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:46:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T06:46:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:46:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:46:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 36436992 bytes 2026-02-18T06:46:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 69224448 bytes 2026-02-18T06:46:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 80095232 bytes 2026-02-18T06:46:33.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 122031104 bytes 2026-02-18T06:46:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 174153728 bytes 2026-02-18T06:46:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 215833600 bytes 2026-02-18T06:46:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 257991680 bytes 2026-02-18T06:46:33.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 301890560 bytes 2026-02-18T06:46:33.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 336375808 bytes 2026-02-18T06:46:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 403330048 bytes 2026-02-18T06:46:33.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 447187968 bytes 2026-02-18T06:46:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 529512448 bytes 2026-02-18T06:46:33.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 604886016 bytes 2026-02-18T06:46:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 645168128 bytes 2026-02-18T06:46:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 691621888 bytes 2026-02-18T06:46:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 738778112 bytes 2026-02-18T06:46:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 802277376 bytes 2026-02-18T06:46:34.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 866580480 bytes 2026-02-18T06:46:34.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 912259072 bytes 2026-02-18T06:46:34.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 952539136 bytes 2026-02-18T06:46:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 974488576 bytes 2026-02-18T06:46:34.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1017790464 bytes 2026-02-18T06:46:34.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1025050624 bytes 2026-02-18T06:46:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1079241728 bytes 2026-02-18T06:46:34.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1099297792 bytes 2026-02-18T06:46:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1153224704 bytes 2026-02-18T06:46:34.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1186058240 bytes 2026-02-18T06:46:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1227088896 bytes 2026-02-18T06:46:34.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1280673792 bytes 2026-02-18T06:46:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1379026944 bytes 2026-02-18T06:46:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1381680128 bytes 2026-02-18T06:46:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1392272384 bytes 2026-02-18T06:46:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1452674048 bytes 2026-02-18T06:46:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1529513984 bytes 2026-02-18T06:46:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1586794496 bytes 2026-02-18T06:46:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1613522944 bytes 2026-02-18T06:46:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1679075328 bytes 2026-02-18T06:46:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1717932032 bytes 2026-02-18T06:46:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1728748544 bytes 2026-02-18T06:46:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1785811968 bytes 2026-02-18T06:46:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1831621632 bytes 2026-02-18T06:46:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1860111360 bytes 2026-02-18T06:46:35.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1949615104 bytes 2026-02-18T06:46:35.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2010177536 bytes 2026-02-18T06:46:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2010817536 bytes 2026-02-18T06:46:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2081980416 bytes 2026-02-18T06:46:35.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2086617088 bytes 2026-02-18T06:46:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2151790592 bytes 2026-02-18T06:46:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2182700032 bytes 2026-02-18T06:46:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2198000640 bytes 2026-02-18T06:46:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2222536704 bytes 2026-02-18T06:46:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2347451392 bytes 2026-02-18T06:46:36.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2367392768 bytes 2026-02-18T06:46:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2401683456 bytes 2026-02-18T06:46:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2440262656 bytes 2026-02-18T06:46:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2491792384 bytes 2026-02-18T06:46:36.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2495955968 bytes 2026-02-18T06:46:36.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2546663424 bytes 2026-02-18T06:46:36.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2583640064 bytes 2026-02-18T06:46:36.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2607938560 bytes 2026-02-18T06:46:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2633691136 bytes 2026-02-18T06:46:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2670419968 bytes 2026-02-18T06:46:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2685076480 bytes 2026-02-18T06:46:36.867 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.334582727721332) > weight (0.25) 2026-02-18T06:46:36.867 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T06:46:36.867 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T06:46:36.868 INFO:tasks.rbd_mirror_thrash:waiting for 3.9520337116870707 secs before reviving daemons 2026-02-18T06:46:36.868 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:46:36.866+0000 7faec3676640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 35917) UID: 1000 2026-02-18T06:46:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2849585152 bytes 2026-02-18T06:46:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2868033536 bytes 2026-02-18T06:46:37.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2907372544 bytes 2026-02-18T06:46:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2909755392 bytes 2026-02-18T06:46:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2910381056 bytes 2026-02-18T06:46:37.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2966017024 bytes 2026-02-18T06:46:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3022074880 bytes 2026-02-18T06:46:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3078990848 bytes 2026-02-18T06:46:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3096044544 bytes 2026-02-18T06:46:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3103964160 bytes 2026-02-18T06:46:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3157803008 bytes 2026-02-18T06:46:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3216154624 bytes 2026-02-18T06:46:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3296505856 bytes 2026-02-18T06:46:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3312345088 bytes 2026-02-18T06:46:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3328843776 bytes 2026-02-18T06:46:37.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3357709312 bytes 2026-02-18T06:46:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3410924544 bytes 2026-02-18T06:46:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3484531712 bytes 2026-02-18T06:46:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3535960064 bytes 2026-02-18T06:46:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3566505984 bytes 2026-02-18T06:46:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3571261440 bytes 2026-02-18T06:46:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3601352704 bytes 2026-02-18T06:46:38.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3612500992 bytes 2026-02-18T06:46:38.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3640346624 bytes 2026-02-18T06:46:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3695414272 bytes 2026-02-18T06:46:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3744309248 bytes 2026-02-18T06:46:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3802271744 bytes 2026-02-18T06:46:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3886701568 bytes 2026-02-18T06:46:38.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3941723136 bytes 2026-02-18T06:46:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4052395008 bytes 2026-02-18T06:46:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4081122304 bytes 2026-02-18T06:46:38.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4142956544 bytes 2026-02-18T06:46:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4193727488 bytes 2026-02-18T06:46:38.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4229444608 bytes 2026-02-18T06:46:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4257868800 bytes 2026-02-18T06:46:38.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4314920960 bytes 2026-02-18T06:46:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4341112832 bytes 2026-02-18T06:46:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4341725184 bytes 2026-02-18T06:46:39.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4414278656 bytes 2026-02-18T06:46:39.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4483347456 bytes 2026-02-18T06:46:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4550748160 bytes 2026-02-18T06:46:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4564625408 bytes 2026-02-18T06:46:39.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4584548352 bytes 2026-02-18T06:46:39.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4585212928 bytes 2026-02-18T06:46:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4585807872 bytes 2026-02-18T06:46:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4604932096 bytes 2026-02-18T06:46:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4630046720 bytes 2026-02-18T06:46:39.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4686141440 bytes 2026-02-18T06:46:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4691660800 bytes 2026-02-18T06:46:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4692278272 bytes 2026-02-18T06:46:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4781213696 bytes 2026-02-18T06:46:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4825018368 bytes 2026-02-18T06:46:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4872813568 bytes 2026-02-18T06:46:39.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4910764032 bytes 2026-02-18T06:46:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4917158912 bytes 2026-02-18T06:46:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4966463488 bytes 2026-02-18T06:46:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5015959552 bytes 2026-02-18T06:46:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5074108416 bytes 2026-02-18T06:46:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5096564736 bytes 2026-02-18T06:46:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5131318272 bytes 2026-02-18T06:46:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5178424320 bytes 2026-02-18T06:46:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5200953344 bytes 2026-02-18T06:46:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5254713344 bytes 2026-02-18T06:46:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5316250624 bytes 2026-02-18T06:46:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5359713280 bytes 2026-02-18T06:46:40.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5391504384 bytes 2026-02-18T06:46:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5440218112 bytes 2026-02-18T06:46:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5486119936 bytes 2026-02-18T06:46:40.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5486698496 bytes 2026-02-18T06:46:40.820 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T06:46:40.821 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:46:40.821 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T06:46:40.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T06:46:40.821 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T06:46:40.823 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T06:46:40.823 INFO:tasks.rbd_mirror_thrash:waiting for 74.16896467164956 secs before thrashing 2026-02-18T06:46:40.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5523159040 bytes 2026-02-18T06:46:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5553043456 bytes 2026-02-18T06:46:40.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5601212416 bytes 2026-02-18T06:46:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5675395072 bytes 2026-02-18T06:46:41.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5787718656 bytes 2026-02-18T06:46:41.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5842265088 bytes 2026-02-18T06:46:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5876278272 bytes 2026-02-18T06:46:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5896947712 bytes 2026-02-18T06:46:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5911875584 bytes 2026-02-18T06:46:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5996988416 bytes 2026-02-18T06:46:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6056489984 bytes 2026-02-18T06:46:41.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6114171904 bytes 2026-02-18T06:46:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6157637632 bytes 2026-02-18T06:46:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6185506816 bytes 2026-02-18T06:46:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6200747008 bytes 2026-02-18T06:46:41.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6221424640 bytes 2026-02-18T06:46:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6222008320 bytes 2026-02-18T06:46:41.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6265416704 bytes 2026-02-18T06:46:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6338497536 bytes 2026-02-18T06:46:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6416104448 bytes 2026-02-18T06:46:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6466985984 bytes 2026-02-18T06:46:42.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6511036416 bytes 2026-02-18T06:46:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6543473664 bytes 2026-02-18T06:46:42.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6610637824 bytes 2026-02-18T06:46:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6664004608 bytes 2026-02-18T06:46:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6674325504 bytes 2026-02-18T06:46:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6700288000 bytes 2026-02-18T06:46:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6749562880 bytes 2026-02-18T06:46:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6763685888 bytes 2026-02-18T06:46:42.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6782005248 bytes 2026-02-18T06:46:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6807571456 bytes 2026-02-18T06:46:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6878296064 bytes 2026-02-18T06:46:42.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6925106176 bytes 2026-02-18T06:46:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6987940864 bytes 2026-02-18T06:46:42.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7015248896 bytes 2026-02-18T06:46:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7018710016 bytes 2026-02-18T06:46:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7064092672 bytes 2026-02-18T06:46:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7143826432 bytes 2026-02-18T06:46:43.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7198259200 bytes 2026-02-18T06:46:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7236020224 bytes 2026-02-18T06:46:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7273585664 bytes 2026-02-18T06:46:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7311461376 bytes 2026-02-18T06:46:43.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7340841984 bytes 2026-02-18T06:46:43.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7386874880 bytes 2026-02-18T06:46:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7458453504 bytes 2026-02-18T06:46:43.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7470097408 bytes 2026-02-18T06:46:43.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7566326784 bytes 2026-02-18T06:46:44.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7681514496 bytes 2026-02-18T06:46:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7752998912 bytes 2026-02-18T06:46:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7877909504 bytes 2026-02-18T06:46:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7948742656 bytes 2026-02-18T06:46:44.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7977463808 bytes 2026-02-18T06:46:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7991974912 bytes 2026-02-18T06:46:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8052796416 bytes 2026-02-18T06:46:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8079184896 bytes 2026-02-18T06:46:44.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8119946240 bytes 2026-02-18T06:46:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8128668672 bytes 2026-02-18T06:46:44.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8209601536 bytes 2026-02-18T06:46:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8261796864 bytes 2026-02-18T06:46:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8334161920 bytes 2026-02-18T06:46:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8334774272 bytes 2026-02-18T06:46:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8335376384 bytes 2026-02-18T06:46:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8343545856 bytes 2026-02-18T06:46:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8357821440 bytes 2026-02-18T06:46:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8358455296 bytes 2026-02-18T06:46:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:46:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:46:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:46:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_6 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:46:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:46:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:46:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_6 2026-02-18T06:46:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:46:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:46:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2026-02-18T06:46:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:46:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-18T06:46:45.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:46:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_6': 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12bc5a6f2fdd 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12bc5a6f2fdd 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:46:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:45 2026 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:45 2026 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:45 2026 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 42f0c001-dae4-46f5-8bba-6fe81293309d 2026-02-18T06:46:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:46:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:46:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-18T06:46:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:46:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:46:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:46:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:46:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:46:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 40071168 bytes 2026-02-18T06:46:46.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 85740544 bytes 2026-02-18T06:46:46.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 119706624 bytes 2026-02-18T06:46:46.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 135720960 bytes 2026-02-18T06:46:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 184670208 bytes 2026-02-18T06:46:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 225050624 bytes 2026-02-18T06:46:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 255433728 bytes 2026-02-18T06:46:46.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 316360704 bytes 2026-02-18T06:46:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 352405504 bytes 2026-02-18T06:46:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 466996224 bytes 2026-02-18T06:46:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 550726656 bytes 2026-02-18T06:46:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 627378176 bytes 2026-02-18T06:46:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 699409408 bytes 2026-02-18T06:46:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 709383168 bytes 2026-02-18T06:46:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 725824512 bytes 2026-02-18T06:46:47.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 778962944 bytes 2026-02-18T06:46:47.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 858285056 bytes 2026-02-18T06:46:47.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 864209920 bytes 2026-02-18T06:46:47.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 949364736 bytes 2026-02-18T06:46:47.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 979954688 bytes 2026-02-18T06:46:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1018508288 bytes 2026-02-18T06:46:47.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1060087808 bytes 2026-02-18T06:46:47.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1146100736 bytes 2026-02-18T06:46:47.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1187111936 bytes 2026-02-18T06:46:47.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1264784384 bytes 2026-02-18T06:46:47.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1297626112 bytes 2026-02-18T06:46:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1338739712 bytes 2026-02-18T06:46:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1356565504 bytes 2026-02-18T06:46:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1410128896 bytes 2026-02-18T06:46:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1488029696 bytes 2026-02-18T06:46:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1506529280 bytes 2026-02-18T06:46:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1517702144 bytes 2026-02-18T06:46:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1564660736 bytes 2026-02-18T06:46:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1597896704 bytes 2026-02-18T06:46:48.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1636662272 bytes 2026-02-18T06:46:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1705527296 bytes 2026-02-18T06:46:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1756778496 bytes 2026-02-18T06:46:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1760873472 bytes 2026-02-18T06:46:48.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1769689088 bytes 2026-02-18T06:46:48.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1825510400 bytes 2026-02-18T06:46:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1831211008 bytes 2026-02-18T06:46:48.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1897798656 bytes 2026-02-18T06:46:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2006112256 bytes 2026-02-18T06:46:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2090062848 bytes 2026-02-18T06:46:48.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2105417728 bytes 2026-02-18T06:46:48.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2151279616 bytes 2026-02-18T06:46:48.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2202135552 bytes 2026-02-18T06:46:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2219160576 bytes 2026-02-18T06:46:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2243657728 bytes 2026-02-18T06:46:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2280588288 bytes 2026-02-18T06:46:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2308799488 bytes 2026-02-18T06:46:49.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2390284288 bytes 2026-02-18T06:46:49.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2410427392 bytes 2026-02-18T06:46:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2496327680 bytes 2026-02-18T06:46:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2523551744 bytes 2026-02-18T06:46:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2638267392 bytes 2026-02-18T06:46:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2648027136 bytes 2026-02-18T06:46:49.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2654316544 bytes 2026-02-18T06:46:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2654929920 bytes 2026-02-18T06:46:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2679561216 bytes 2026-02-18T06:46:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2685824000 bytes 2026-02-18T06:46:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2788274176 bytes 2026-02-18T06:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2797404160 bytes 2026-02-18T06:46:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2832307200 bytes 2026-02-18T06:46:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2881786880 bytes 2026-02-18T06:46:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2918609920 bytes 2026-02-18T06:46:50.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2919228416 bytes 2026-02-18T06:46:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2942088192 bytes 2026-02-18T06:46:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2962319360 bytes 2026-02-18T06:46:50.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3020359680 bytes 2026-02-18T06:46:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3061757952 bytes 2026-02-18T06:46:50.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3065809920 bytes 2026-02-18T06:46:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3127166976 bytes 2026-02-18T06:46:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3189324800 bytes 2026-02-18T06:46:50.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3194690560 bytes 2026-02-18T06:46:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3217667072 bytes 2026-02-18T06:46:50.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3268035584 bytes 2026-02-18T06:46:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3306480640 bytes 2026-02-18T06:46:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3401383936 bytes 2026-02-18T06:46:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3443733504 bytes 2026-02-18T06:46:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3511435264 bytes 2026-02-18T06:46:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3547179008 bytes 2026-02-18T06:46:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3599295488 bytes 2026-02-18T06:46:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3624463360 bytes 2026-02-18T06:46:51.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3670696960 bytes 2026-02-18T06:46:51.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3737800704 bytes 2026-02-18T06:46:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3764238336 bytes 2026-02-18T06:46:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3805716480 bytes 2026-02-18T06:46:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3862916096 bytes 2026-02-18T06:46:51.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3918842880 bytes 2026-02-18T06:46:51.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3961160704 bytes 2026-02-18T06:46:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3978058752 bytes 2026-02-18T06:46:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4034980864 bytes 2026-02-18T06:46:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4050557952 bytes 2026-02-18T06:46:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4094078976 bytes 2026-02-18T06:46:51.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4126788608 bytes 2026-02-18T06:46:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4143268864 bytes 2026-02-18T06:46:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4178932736 bytes 2026-02-18T06:46:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4228664320 bytes 2026-02-18T06:46:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4311591936 bytes 2026-02-18T06:46:52.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4342639616 bytes 2026-02-18T06:46:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4363279360 bytes 2026-02-18T06:46:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4413196288 bytes 2026-02-18T06:46:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4501204992 bytes 2026-02-18T06:46:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4520130560 bytes 2026-02-18T06:46:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4562876416 bytes 2026-02-18T06:46:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4572975104 bytes 2026-02-18T06:46:52.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4573590528 bytes 2026-02-18T06:46:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4597559296 bytes 2026-02-18T06:46:52.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4662811648 bytes 2026-02-18T06:46:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4707373056 bytes 2026-02-18T06:46:52.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4735538176 bytes 2026-02-18T06:46:52.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4776308736 bytes 2026-02-18T06:46:52.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4836616192 bytes 2026-02-18T06:46:53.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4860467200 bytes 2026-02-18T06:46:53.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4950668288 bytes 2026-02-18T06:46:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5026615296 bytes 2026-02-18T06:46:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5028587520 bytes 2026-02-18T06:46:53.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5062781952 bytes 2026-02-18T06:46:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5101070336 bytes 2026-02-18T06:46:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5101702144 bytes 2026-02-18T06:46:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5131547648 bytes 2026-02-18T06:46:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5176141824 bytes 2026-02-18T06:46:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5176762368 bytes 2026-02-18T06:46:53.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5255219200 bytes 2026-02-18T06:46:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5388870656 bytes 2026-02-18T06:46:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5431452672 bytes 2026-02-18T06:46:53.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5436243968 bytes 2026-02-18T06:46:53.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5513317376 bytes 2026-02-18T06:46:53.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5569356800 bytes 2026-02-18T06:46:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5634045952 bytes 2026-02-18T06:46:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5686462464 bytes 2026-02-18T06:46:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5716017152 bytes 2026-02-18T06:46:54.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5758212096 bytes 2026-02-18T06:46:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5785196544 bytes 2026-02-18T06:46:54.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5807436800 bytes 2026-02-18T06:46:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5813758976 bytes 2026-02-18T06:46:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5919362048 bytes 2026-02-18T06:46:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5939196928 bytes 2026-02-18T06:46:54.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5980867584 bytes 2026-02-18T06:46:54.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6033386496 bytes 2026-02-18T06:46:54.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6066173952 bytes 2026-02-18T06:46:54.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6116482048 bytes 2026-02-18T06:46:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6158174208 bytes 2026-02-18T06:46:54.940 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T06:46:54.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T06:46:54.941 INFO:tasks.rbd_mirror_thrash:waiting for 0.6609741718626683 secs before reviving daemons 2026-02-18T06:46:54.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T06:46:54.940+0000 7ff0007de640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 37862) UID: 1000 2026-02-18T06:46:54.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6241971200 bytes 2026-02-18T06:46:55.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6308768768 bytes 2026-02-18T06:46:55.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6317369344 bytes 2026-02-18T06:46:55.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6392682496 bytes 2026-02-18T06:46:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6432571392 bytes 2026-02-18T06:46:55.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6494909440 bytes 2026-02-18T06:46:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6495553536 bytes 2026-02-18T06:46:55.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6550910976 bytes 2026-02-18T06:46:55.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6598179840 bytes 2026-02-18T06:46:55.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6648468480 bytes 2026-02-18T06:46:55.603 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T06:46:55.603 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:46:55.603 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T06:46:55.603 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:46:55.603 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:46:55.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:46:55.606 INFO:tasks.rbd_mirror_thrash:waiting for 114.24910582365698 secs before thrashing 2026-02-18T06:46:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6704656384 bytes 2026-02-18T06:46:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6763756544 bytes 2026-02-18T06:46:55.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6775777280 bytes 2026-02-18T06:46:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6831695872 bytes 2026-02-18T06:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6905663488 bytes 2026-02-18T06:46:55.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6963576832 bytes 2026-02-18T06:46:56.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6991585280 bytes 2026-02-18T06:46:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7080921088 bytes 2026-02-18T06:46:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7115229184 bytes 2026-02-18T06:46:56.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7116422144 bytes 2026-02-18T06:46:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7198917632 bytes 2026-02-18T06:46:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7199529984 bytes 2026-02-18T06:46:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7241247744 bytes 2026-02-18T06:46:56.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7260217344 bytes 2026-02-18T06:46:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7321064448 bytes 2026-02-18T06:46:56.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7342922752 bytes 2026-02-18T06:46:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7413700608 bytes 2026-02-18T06:46:56.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7462529024 bytes 2026-02-18T06:46:56.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7479723008 bytes 2026-02-18T06:46:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7520061440 bytes 2026-02-18T06:46:56.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7520679936 bytes 2026-02-18T06:46:56.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7571264512 bytes 2026-02-18T06:46:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7590614016 bytes 2026-02-18T06:46:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7635244032 bytes 2026-02-18T06:46:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7649918976 bytes 2026-02-18T06:46:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7694279680 bytes 2026-02-18T06:46:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7713929216 bytes 2026-02-18T06:46:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7734959104 bytes 2026-02-18T06:46:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7773914112 bytes 2026-02-18T06:46:57.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7819294720 bytes 2026-02-18T06:46:57.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7847745536 bytes 2026-02-18T06:46:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7877577728 bytes 2026-02-18T06:46:57.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7932391424 bytes 2026-02-18T06:46:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8018057216 bytes 2026-02-18T06:46:57.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8045679616 bytes 2026-02-18T06:46:57.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8099791872 bytes 2026-02-18T06:46:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8127819776 bytes 2026-02-18T06:46:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8158231552 bytes 2026-02-18T06:46:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8209883136 bytes 2026-02-18T06:46:58.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8317490176 bytes 2026-02-18T06:46:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8350375936 bytes 2026-02-18T06:46:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8406079488 bytes 2026-02-18T06:46:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8428118016 bytes 2026-02-18T06:46:58.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8483530752 bytes 2026-02-18T06:46:58.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8533797888 bytes 2026-02-18T06:46:58.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8568033280 bytes 2026-02-18T06:46:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8617082880 bytes 2026-02-18T06:46:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8630812672 bytes 2026-02-18T06:46:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8679019520 bytes 2026-02-18T06:46:58.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8748104704 bytes 2026-02-18T06:46:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8796092416 bytes 2026-02-18T06:46:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8901193728 bytes 2026-02-18T06:46:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8982246400 bytes 2026-02-18T06:46:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:46:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:46:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:46:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:59.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_7 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:46:59.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:46:59.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_7 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:46:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:46:59.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-18T06:46:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:46:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-18T06:47:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_7': 2026-02-18T06:47:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12d047e1785f 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12d047e1785f 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:59 2026 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:59 2026 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:59 2026 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 21e2d0de-af6e-45ad-b644-f4eee184726c 2026-02-18T06:47:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-18T06:47:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:47:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:47:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 36475904 bytes 2026-02-18T06:47:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 91172864 bytes 2026-02-18T06:47:00.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 123385856 bytes 2026-02-18T06:47:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 160586752 bytes 2026-02-18T06:47:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 161222656 bytes 2026-02-18T06:47:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 207320064 bytes 2026-02-18T06:47:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 230372352 bytes 2026-02-18T06:47:00.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 308279296 bytes 2026-02-18T06:47:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 328719360 bytes 2026-02-18T06:47:00.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 367970304 bytes 2026-02-18T06:47:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 377089024 bytes 2026-02-18T06:47:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 448989184 bytes 2026-02-18T06:47:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 471176192 bytes 2026-02-18T06:47:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 513155072 bytes 2026-02-18T06:47:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 540240896 bytes 2026-02-18T06:47:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 551347200 bytes 2026-02-18T06:47:01.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 594604032 bytes 2026-02-18T06:47:01.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 702367744 bytes 2026-02-18T06:47:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 726273024 bytes 2026-02-18T06:47:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 746478592 bytes 2026-02-18T06:47:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 825290752 bytes 2026-02-18T06:47:01.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 846384128 bytes 2026-02-18T06:47:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 901568512 bytes 2026-02-18T06:47:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 926492672 bytes 2026-02-18T06:47:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 972098560 bytes 2026-02-18T06:47:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1014375424 bytes 2026-02-18T06:47:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1046355968 bytes 2026-02-18T06:47:01.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1099637760 bytes 2026-02-18T06:47:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1167559680 bytes 2026-02-18T06:47:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1223045120 bytes 2026-02-18T06:47:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1243944960 bytes 2026-02-18T06:47:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1291887616 bytes 2026-02-18T06:47:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1292514304 bytes 2026-02-18T06:47:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1396166656 bytes 2026-02-18T06:47:02.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1396789248 bytes 2026-02-18T06:47:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1428267008 bytes 2026-02-18T06:47:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1501632512 bytes 2026-02-18T06:47:02.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1522835456 bytes 2026-02-18T06:47:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1618215936 bytes 2026-02-18T06:47:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1642823680 bytes 2026-02-18T06:47:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1710993408 bytes 2026-02-18T06:47:02.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1728878592 bytes 2026-02-18T06:47:02.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1773298688 bytes 2026-02-18T06:47:02.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1811911680 bytes 2026-02-18T06:47:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1846470656 bytes 2026-02-18T06:47:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1908861952 bytes 2026-02-18T06:47:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1954317312 bytes 2026-02-18T06:47:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2005990400 bytes 2026-02-18T06:47:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2091893760 bytes 2026-02-18T06:47:03.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2153227264 bytes 2026-02-18T06:47:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2222952448 bytes 2026-02-18T06:47:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2263880704 bytes 2026-02-18T06:47:03.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2285608960 bytes 2026-02-18T06:47:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2321235968 bytes 2026-02-18T06:47:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2357232640 bytes 2026-02-18T06:47:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2421535744 bytes 2026-02-18T06:47:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2534078464 bytes 2026-02-18T06:47:03.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2549708800 bytes 2026-02-18T06:47:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2662076416 bytes 2026-02-18T06:47:03.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2667361280 bytes 2026-02-18T06:47:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2719370240 bytes 2026-02-18T06:47:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2782911488 bytes 2026-02-18T06:47:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2830190592 bytes 2026-02-18T06:47:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2868349952 bytes 2026-02-18T06:47:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2917714944 bytes 2026-02-18T06:47:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2933128192 bytes 2026-02-18T06:47:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2963160064 bytes 2026-02-18T06:47:04.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2963801088 bytes 2026-02-18T06:47:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3033036800 bytes 2026-02-18T06:47:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3079370752 bytes 2026-02-18T06:47:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3116541952 bytes 2026-02-18T06:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3168143360 bytes 2026-02-18T06:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3215107072 bytes 2026-02-18T06:47:04.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3252398080 bytes 2026-02-18T06:47:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3266430976 bytes 2026-02-18T06:47:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3291726848 bytes 2026-02-18T06:47:04.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3324483584 bytes 2026-02-18T06:47:04.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3327913984 bytes 2026-02-18T06:47:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3352142848 bytes 2026-02-18T06:47:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3434801152 bytes 2026-02-18T06:47:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3466394624 bytes 2026-02-18T06:47:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3486294016 bytes 2026-02-18T06:47:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3533955072 bytes 2026-02-18T06:47:05.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3544696832 bytes 2026-02-18T06:47:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3607998464 bytes 2026-02-18T06:47:05.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3658483712 bytes 2026-02-18T06:47:05.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3700688896 bytes 2026-02-18T06:47:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3739180032 bytes 2026-02-18T06:47:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3751426048 bytes 2026-02-18T06:47:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3761395712 bytes 2026-02-18T06:47:05.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3816315904 bytes 2026-02-18T06:47:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3851640832 bytes 2026-02-18T06:47:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3933825024 bytes 2026-02-18T06:47:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4006990848 bytes 2026-02-18T06:47:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4021459968 bytes 2026-02-18T06:47:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4089296896 bytes 2026-02-18T06:47:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:47:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:47:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:47:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_8 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:06.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:47:06.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:47:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_8 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:47:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-18T06:47:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:47:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-18T06:47:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_8': 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12dc2cd6650f 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12dc2cd6650f 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:06 2026 2026-02-18T06:47:07.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:06 2026 2026-02-18T06:47:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:06 2026 2026-02-18T06:47:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 19f68516-8b2a-45d7-b3fb-465c882305ae 2026-02-18T06:47:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:47:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:47:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 45105152 bytes 2026-02-18T06:47:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 72925184 bytes 2026-02-18T06:47:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 97454080 bytes 2026-02-18T06:47:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 102397952 bytes 2026-02-18T06:47:07.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 201556992 bytes 2026-02-18T06:47:07.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 202129408 bytes 2026-02-18T06:47:07.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 232280064 bytes 2026-02-18T06:47:07.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 256650240 bytes 2026-02-18T06:47:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 295378944 bytes 2026-02-18T06:47:07.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 330263552 bytes 2026-02-18T06:47:07.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 379587584 bytes 2026-02-18T06:47:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 408699904 bytes 2026-02-18T06:47:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 439615488 bytes 2026-02-18T06:47:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 473970688 bytes 2026-02-18T06:47:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 495718400 bytes 2026-02-18T06:47:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 534921216 bytes 2026-02-18T06:47:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 559865856 bytes 2026-02-18T06:47:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 606921728 bytes 2026-02-18T06:47:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 703935488 bytes 2026-02-18T06:47:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 760443904 bytes 2026-02-18T06:47:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 761047040 bytes 2026-02-18T06:47:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 817161216 bytes 2026-02-18T06:47:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 876859392 bytes 2026-02-18T06:47:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 903658496 bytes 2026-02-18T06:47:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 943805440 bytes 2026-02-18T06:47:08.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 989034496 bytes 2026-02-18T06:47:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1031593984 bytes 2026-02-18T06:47:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1045405696 bytes 2026-02-18T06:47:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1146999808 bytes 2026-02-18T06:47:08.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1147602944 bytes 2026-02-18T06:47:08.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1169671168 bytes 2026-02-18T06:47:08.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1170287616 bytes 2026-02-18T06:47:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1224536064 bytes 2026-02-18T06:47:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1235307520 bytes 2026-02-18T06:47:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1294691328 bytes 2026-02-18T06:47:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1348022272 bytes 2026-02-18T06:47:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1438589952 bytes 2026-02-18T06:47:09.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1458200576 bytes 2026-02-18T06:47:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1476429824 bytes 2026-02-18T06:47:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1496039424 bytes 2026-02-18T06:47:09.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1517904896 bytes 2026-02-18T06:47:09.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1608393728 bytes 2026-02-18T06:47:09.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1626471424 bytes 2026-02-18T06:47:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1658824704 bytes 2026-02-18T06:47:09.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1701170176 bytes 2026-02-18T06:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1761146880 bytes 2026-02-18T06:47:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1793167360 bytes 2026-02-18T06:47:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1861457920 bytes 2026-02-18T06:47:09.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1914387456 bytes 2026-02-18T06:47:09.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1983663104 bytes 2026-02-18T06:47:10.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2019718144 bytes 2026-02-18T06:47:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2058584064 bytes 2026-02-18T06:47:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2105350144 bytes 2026-02-18T06:47:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2142039040 bytes 2026-02-18T06:47:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2152473600 bytes 2026-02-18T06:47:10.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2213411840 bytes 2026-02-18T06:47:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2309014528 bytes 2026-02-18T06:47:10.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2352790528 bytes 2026-02-18T06:47:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2389952512 bytes 2026-02-18T06:47:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2444748800 bytes 2026-02-18T06:47:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2497603584 bytes 2026-02-18T06:47:10.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2591383552 bytes 2026-02-18T06:47:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2615139328 bytes 2026-02-18T06:47:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2646174720 bytes 2026-02-18T06:47:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2659477504 bytes 2026-02-18T06:47:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2760674304 bytes 2026-02-18T06:47:11.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2827543552 bytes 2026-02-18T06:47:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2876441600 bytes 2026-02-18T06:47:11.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2934355968 bytes 2026-02-18T06:47:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2937903104 bytes 2026-02-18T06:47:11.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2979623936 bytes 2026-02-18T06:47:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3026765824 bytes 2026-02-18T06:47:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3129490432 bytes 2026-02-18T06:47:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3157063680 bytes 2026-02-18T06:47:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3203869696 bytes 2026-02-18T06:47:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3251606528 bytes 2026-02-18T06:47:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3294425088 bytes 2026-02-18T06:47:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3327360000 bytes 2026-02-18T06:47:11.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3366961152 bytes 2026-02-18T06:47:11.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3405755392 bytes 2026-02-18T06:47:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3438478336 bytes 2026-02-18T06:47:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3442641920 bytes 2026-02-18T06:47:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3452692480 bytes 2026-02-18T06:47:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3551921152 bytes 2026-02-18T06:47:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3610504192 bytes 2026-02-18T06:47:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3627860992 bytes 2026-02-18T06:47:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3686643712 bytes 2026-02-18T06:47:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3735980032 bytes 2026-02-18T06:47:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3794351104 bytes 2026-02-18T06:47:12.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3794953216 bytes 2026-02-18T06:47:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3801364480 bytes 2026-02-18T06:47:12.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3802793984 bytes 2026-02-18T06:47:12.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3821028352 bytes 2026-02-18T06:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3896202240 bytes 2026-02-18T06:47:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3918806016 bytes 2026-02-18T06:47:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3919421440 bytes 2026-02-18T06:47:12.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3942802432 bytes 2026-02-18T06:47:12.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3966631936 bytes 2026-02-18T06:47:12.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4052762624 bytes 2026-02-18T06:47:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4103586816 bytes 2026-02-18T06:47:13.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4151957504 bytes 2026-02-18T06:47:13.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4227774464 bytes 2026-02-18T06:47:13.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4298591232 bytes 2026-02-18T06:47:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4322184192 bytes 2026-02-18T06:47:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4338219008 bytes 2026-02-18T06:47:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4366670848 bytes 2026-02-18T06:47:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4416211968 bytes 2026-02-18T06:47:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4456207360 bytes 2026-02-18T06:47:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4487176192 bytes 2026-02-18T06:47:13.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4511755264 bytes 2026-02-18T06:47:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4557459456 bytes 2026-02-18T06:47:13.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4639271936 bytes 2026-02-18T06:47:13.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4744458240 bytes 2026-02-18T06:47:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4796738560 bytes 2026-02-18T06:47:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4797309952 bytes 2026-02-18T06:47:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4860389376 bytes 2026-02-18T06:47:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4885288960 bytes 2026-02-18T06:47:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4990740480 bytes 2026-02-18T06:47:14.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5056754688 bytes 2026-02-18T06:47:14.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5089144832 bytes 2026-02-18T06:47:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5152192512 bytes 2026-02-18T06:47:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5193316352 bytes 2026-02-18T06:47:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5264657408 bytes 2026-02-18T06:47:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5311946752 bytes 2026-02-18T06:47:14.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5346067456 bytes 2026-02-18T06:47:14.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5384633344 bytes 2026-02-18T06:47:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5432953856 bytes 2026-02-18T06:47:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5438662656 bytes 2026-02-18T06:47:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5452944384 bytes 2026-02-18T06:47:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5474675712 bytes 2026-02-18T06:47:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5530939392 bytes 2026-02-18T06:47:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5565359104 bytes 2026-02-18T06:47:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5639012352 bytes 2026-02-18T06:47:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5682856960 bytes 2026-02-18T06:47:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5736218624 bytes 2026-02-18T06:47:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5777038336 bytes 2026-02-18T06:47:15.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5835928576 bytes 2026-02-18T06:47:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5840215040 bytes 2026-02-18T06:47:15.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5886820352 bytes 2026-02-18T06:47:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5920732160 bytes 2026-02-18T06:47:15.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6012023808 bytes 2026-02-18T06:47:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6030749696 bytes 2026-02-18T06:47:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6041645056 bytes 2026-02-18T06:47:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6086339584 bytes 2026-02-18T06:47:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6110344192 bytes 2026-02-18T06:47:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6121331712 bytes 2026-02-18T06:47:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6137562112 bytes 2026-02-18T06:47:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6169278464 bytes 2026-02-18T06:47:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6181765120 bytes 2026-02-18T06:47:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6233408512 bytes 2026-02-18T06:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6312295424 bytes 2026-02-18T06:47:16.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6341046272 bytes 2026-02-18T06:47:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6400231424 bytes 2026-02-18T06:47:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6495523840 bytes 2026-02-18T06:47:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6520608768 bytes 2026-02-18T06:47:16.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6544826368 bytes 2026-02-18T06:47:16.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6576019456 bytes 2026-02-18T06:47:16.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6596145152 bytes 2026-02-18T06:47:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6633872384 bytes 2026-02-18T06:47:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6655650816 bytes 2026-02-18T06:47:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6681299968 bytes 2026-02-18T06:47:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6717368320 bytes 2026-02-18T06:47:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6773648384 bytes 2026-02-18T06:47:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6844621824 bytes 2026-02-18T06:47:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6855343104 bytes 2026-02-18T06:47:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6888242176 bytes 2026-02-18T06:47:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6925178880 bytes 2026-02-18T06:47:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6951418880 bytes 2026-02-18T06:47:17.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6972459008 bytes 2026-02-18T06:47:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6987432960 bytes 2026-02-18T06:47:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6988059648 bytes 2026-02-18T06:47:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6988677120 bytes 2026-02-18T06:47:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6998669312 bytes 2026-02-18T06:47:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7017862144 bytes 2026-02-18T06:47:17.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7059334144 bytes 2026-02-18T06:47:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7063330816 bytes 2026-02-18T06:47:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7103221760 bytes 2026-02-18T06:47:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7126778880 bytes 2026-02-18T06:47:17.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7161725952 bytes 2026-02-18T06:47:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7204193280 bytes 2026-02-18T06:47:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7244005376 bytes 2026-02-18T06:47:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7282058240 bytes 2026-02-18T06:47:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7323215872 bytes 2026-02-18T06:47:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7380236288 bytes 2026-02-18T06:47:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7420098560 bytes 2026-02-18T06:47:17.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7455594496 bytes 2026-02-18T06:47:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7474779136 bytes 2026-02-18T06:47:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7570831360 bytes 2026-02-18T06:47:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7592004608 bytes 2026-02-18T06:47:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7607607296 bytes 2026-02-18T06:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7637108736 bytes 2026-02-18T06:47:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7677726720 bytes 2026-02-18T06:47:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7721356288 bytes 2026-02-18T06:47:18.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7731696640 bytes 2026-02-18T06:47:18.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7768154112 bytes 2026-02-18T06:47:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7871608832 bytes 2026-02-18T06:47:18.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7898569728 bytes 2026-02-18T06:47:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 7932564480 bytes 2026-02-18T06:47:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 7946156032 bytes 2026-02-18T06:47:18.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 7981886464 bytes 2026-02-18T06:47:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8030844928 bytes 2026-02-18T06:47:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8073993216 bytes 2026-02-18T06:47:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8110838784 bytes 2026-02-18T06:47:19.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8182059008 bytes 2026-02-18T06:47:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8184726528 bytes 2026-02-18T06:47:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8246684672 bytes 2026-02-18T06:47:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8289948672 bytes 2026-02-18T06:47:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8360751104 bytes 2026-02-18T06:47:19.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8400517120 bytes 2026-02-18T06:47:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8524852224 bytes 2026-02-18T06:47:19.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8537901056 bytes 2026-02-18T06:47:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8538503168 bytes 2026-02-18T06:47:19.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8557253632 bytes 2026-02-18T06:47:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 8603335680 bytes 2026-02-18T06:47:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 8642033664 bytes 2026-02-18T06:47:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 8642636800 bytes 2026-02-18T06:47:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 8693687296 bytes 2026-02-18T06:47:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 8769792000 bytes 2026-02-18T06:47:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 8782675968 bytes 2026-02-18T06:47:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 8804860928 bytes 2026-02-18T06:47:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 8851278848 bytes 2026-02-18T06:47:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 8878274560 bytes 2026-02-18T06:47:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 8894670848 bytes 2026-02-18T06:47:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 8966673408 bytes 2026-02-18T06:47:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 8967289856 bytes 2026-02-18T06:47:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 8989889536 bytes 2026-02-18T06:47:20.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 9056113664 bytes 2026-02-18T06:47:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 9083406336 bytes 2026-02-18T06:47:20.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 9183802368 bytes 2026-02-18T06:47:20.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23000 IOs, 9228271616 bytes 2026-02-18T06:47:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23100 IOs, 9270918144 bytes 2026-02-18T06:47:20.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23200 IOs, 9295980544 bytes 2026-02-18T06:47:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_9 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:47:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:47:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:47:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:47:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_9 2026-02-18T06:47:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:47:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:47:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2026-02-18T06:47:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:47:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-18T06:47:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:47:22.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_9': 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12e88f898183 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12e88f898183 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:21 2026 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:21 2026 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:21 2026 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 40b4e69c-a609-443c-8267-78314c9f7989 2026-02-18T06:47:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:47:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:22.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T06:47:22.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:22.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 55464960 bytes 2026-02-18T06:47:22.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 112913408 bytes 2026-02-18T06:47:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 157366272 bytes 2026-02-18T06:47:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 191080448 bytes 2026-02-18T06:47:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 232030208 bytes 2026-02-18T06:47:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 282187776 bytes 2026-02-18T06:47:22.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 300768256 bytes 2026-02-18T06:47:22.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 320278528 bytes 2026-02-18T06:47:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 321382400 bytes 2026-02-18T06:47:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 346171392 bytes 2026-02-18T06:47:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 387344384 bytes 2026-02-18T06:47:22.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 408697856 bytes 2026-02-18T06:47:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 450653184 bytes 2026-02-18T06:47:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 521485312 bytes 2026-02-18T06:47:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 555727872 bytes 2026-02-18T06:47:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 602755072 bytes 2026-02-18T06:47:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 665743360 bytes 2026-02-18T06:47:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 715035648 bytes 2026-02-18T06:47:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 812355584 bytes 2026-02-18T06:47:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 829204480 bytes 2026-02-18T06:47:23.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 876858368 bytes 2026-02-18T06:47:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 894641152 bytes 2026-02-18T06:47:23.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 975483904 bytes 2026-02-18T06:47:23.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1071809536 bytes 2026-02-18T06:47:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1136437248 bytes 2026-02-18T06:47:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1153041408 bytes 2026-02-18T06:47:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1178167296 bytes 2026-02-18T06:47:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1178781696 bytes 2026-02-18T06:47:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1259886592 bytes 2026-02-18T06:47:23.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1283066880 bytes 2026-02-18T06:47:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1326551040 bytes 2026-02-18T06:47:23.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1352835072 bytes 2026-02-18T06:47:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1374175232 bytes 2026-02-18T06:47:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1428896768 bytes 2026-02-18T06:47:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1540611072 bytes 2026-02-18T06:47:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1623209984 bytes 2026-02-18T06:47:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1650548736 bytes 2026-02-18T06:47:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1668379648 bytes 2026-02-18T06:47:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1778594816 bytes 2026-02-18T06:47:24.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1813688320 bytes 2026-02-18T06:47:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1888794624 bytes 2026-02-18T06:47:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1938409472 bytes 2026-02-18T06:47:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1977043968 bytes 2026-02-18T06:47:24.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1984447488 bytes 2026-02-18T06:47:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1999713280 bytes 2026-02-18T06:47:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2073815040 bytes 2026-02-18T06:47:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2149937152 bytes 2026-02-18T06:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2199945216 bytes 2026-02-18T06:47:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2225324032 bytes 2026-02-18T06:47:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2287054848 bytes 2026-02-18T06:47:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2338648064 bytes 2026-02-18T06:47:25.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2366831616 bytes 2026-02-18T06:47:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2393560064 bytes 2026-02-18T06:47:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2394175488 bytes 2026-02-18T06:47:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2432117760 bytes 2026-02-18T06:47:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2436093952 bytes 2026-02-18T06:47:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2513273856 bytes 2026-02-18T06:47:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2599567360 bytes 2026-02-18T06:47:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2706900992 bytes 2026-02-18T06:47:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2715687936 bytes 2026-02-18T06:47:25.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2781352960 bytes 2026-02-18T06:47:25.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2795515904 bytes 2026-02-18T06:47:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2846902272 bytes 2026-02-18T06:47:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2872628224 bytes 2026-02-18T06:47:25.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2909121536 bytes 2026-02-18T06:47:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2944082944 bytes 2026-02-18T06:47:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2986898432 bytes 2026-02-18T06:47:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3096305664 bytes 2026-02-18T06:47:26.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3140499456 bytes 2026-02-18T06:47:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3155650560 bytes 2026-02-18T06:47:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3200528384 bytes 2026-02-18T06:47:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3273608192 bytes 2026-02-18T06:47:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3335442432 bytes 2026-02-18T06:47:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3379627008 bytes 2026-02-18T06:47:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3390473216 bytes 2026-02-18T06:47:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3451402240 bytes 2026-02-18T06:47:26.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3469736960 bytes 2026-02-18T06:47:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3520205824 bytes 2026-02-18T06:47:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3520821248 bytes 2026-02-18T06:47:26.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3579022336 bytes 2026-02-18T06:47:26.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3601036288 bytes 2026-02-18T06:47:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3601651712 bytes 2026-02-18T06:47:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3646612480 bytes 2026-02-18T06:47:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3697930240 bytes 2026-02-18T06:47:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3728822272 bytes 2026-02-18T06:47:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3775407104 bytes 2026-02-18T06:47:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3798801408 bytes 2026-02-18T06:47:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3846542336 bytes 2026-02-18T06:47:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3909779456 bytes 2026-02-18T06:47:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3910394880 bytes 2026-02-18T06:47:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3960314880 bytes 2026-02-18T06:47:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4017489920 bytes 2026-02-18T06:47:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4054142976 bytes 2026-02-18T06:47:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4089174016 bytes 2026-02-18T06:47:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4139947008 bytes 2026-02-18T06:47:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4143503360 bytes 2026-02-18T06:47:27.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4198369280 bytes 2026-02-18T06:47:27.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4204412928 bytes 2026-02-18T06:47:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4213120000 bytes 2026-02-18T06:47:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4248520704 bytes 2026-02-18T06:47:27.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4292311040 bytes 2026-02-18T06:47:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4340622336 bytes 2026-02-18T06:47:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4411491328 bytes 2026-02-18T06:47:28.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4433148928 bytes 2026-02-18T06:47:28.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4501895168 bytes 2026-02-18T06:47:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4558447616 bytes 2026-02-18T06:47:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4644667392 bytes 2026-02-18T06:47:28.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4658916352 bytes 2026-02-18T06:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4680115200 bytes 2026-02-18T06:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4720292864 bytes 2026-02-18T06:47:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4741502976 bytes 2026-02-18T06:47:28.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4806043648 bytes 2026-02-18T06:47:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4815835136 bytes 2026-02-18T06:47:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4830609408 bytes 2026-02-18T06:47:28.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4853448704 bytes 2026-02-18T06:47:28.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4889725952 bytes 2026-02-18T06:47:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4968590336 bytes 2026-02-18T06:47:28.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4999106560 bytes 2026-02-18T06:47:28.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5029690368 bytes 2026-02-18T06:47:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5030283264 bytes 2026-02-18T06:47:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5114888192 bytes 2026-02-18T06:47:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5206386688 bytes 2026-02-18T06:47:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5268510720 bytes 2026-02-18T06:47:29.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5351042048 bytes 2026-02-18T06:47:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5388115968 bytes 2026-02-18T06:47:29.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5453797376 bytes 2026-02-18T06:47:29.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5528818688 bytes 2026-02-18T06:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5542355968 bytes 2026-02-18T06:47:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5564172288 bytes 2026-02-18T06:47:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5620091904 bytes 2026-02-18T06:47:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5648411648 bytes 2026-02-18T06:47:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5698880512 bytes 2026-02-18T06:47:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5727049728 bytes 2026-02-18T06:47:29.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5755631616 bytes 2026-02-18T06:47:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5756249088 bytes 2026-02-18T06:47:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5853766656 bytes 2026-02-18T06:47:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5931087872 bytes 2026-02-18T06:47:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5979145216 bytes 2026-02-18T06:47:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5998036992 bytes 2026-02-18T06:47:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6045914112 bytes 2026-02-18T06:47:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6100890624 bytes 2026-02-18T06:47:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6178592768 bytes 2026-02-18T06:47:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6234475520 bytes 2026-02-18T06:47:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6245854208 bytes 2026-02-18T06:47:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6282566656 bytes 2026-02-18T06:47:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6289893376 bytes 2026-02-18T06:47:30.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6332042240 bytes 2026-02-18T06:47:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6395720704 bytes 2026-02-18T06:47:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6428915712 bytes 2026-02-18T06:47:30.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6494327808 bytes 2026-02-18T06:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6555727872 bytes 2026-02-18T06:47:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6604719104 bytes 2026-02-18T06:47:31.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6632677376 bytes 2026-02-18T06:47:31.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6673596416 bytes 2026-02-18T06:47:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6728996864 bytes 2026-02-18T06:47:31.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6798631936 bytes 2026-02-18T06:47:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6834398208 bytes 2026-02-18T06:47:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6912628736 bytes 2026-02-18T06:47:31.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6944497664 bytes 2026-02-18T06:47:31.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6970046464 bytes 2026-02-18T06:47:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7024676864 bytes 2026-02-18T06:47:31.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7105923072 bytes 2026-02-18T06:47:31.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7135617024 bytes 2026-02-18T06:47:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7162248192 bytes 2026-02-18T06:47:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7230478336 bytes 2026-02-18T06:47:31.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7295093760 bytes 2026-02-18T06:47:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7313702912 bytes 2026-02-18T06:47:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7326304256 bytes 2026-02-18T06:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7337790464 bytes 2026-02-18T06:47:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7374579712 bytes 2026-02-18T06:47:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7398410240 bytes 2026-02-18T06:47:32.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7399033856 bytes 2026-02-18T06:47:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7472555008 bytes 2026-02-18T06:47:32.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7506126848 bytes 2026-02-18T06:47:32.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7570159616 bytes 2026-02-18T06:47:32.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7648232448 bytes 2026-02-18T06:47:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7717222400 bytes 2026-02-18T06:47:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7753068544 bytes 2026-02-18T06:47:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7805400064 bytes 2026-02-18T06:47:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7817485312 bytes 2026-02-18T06:47:32.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7861526528 bytes 2026-02-18T06:47:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7902179328 bytes 2026-02-18T06:47:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7915773952 bytes 2026-02-18T06:47:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7916380160 bytes 2026-02-18T06:47:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7954029568 bytes 2026-02-18T06:47:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8017327104 bytes 2026-02-18T06:47:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8081758208 bytes 2026-02-18T06:47:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8084734976 bytes 2026-02-18T06:47:33.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8158675968 bytes 2026-02-18T06:47:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8180984832 bytes 2026-02-18T06:47:33.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8196407296 bytes 2026-02-18T06:47:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8197016576 bytes 2026-02-18T06:47:33.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8216115200 bytes 2026-02-18T06:47:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8274098176 bytes 2026-02-18T06:47:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8329708544 bytes 2026-02-18T06:47:33.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8364769280 bytes 2026-02-18T06:47:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8374156288 bytes 2026-02-18T06:47:33.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8385098752 bytes 2026-02-18T06:47:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8404128768 bytes 2026-02-18T06:47:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8475659264 bytes 2026-02-18T06:47:33.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8511675392 bytes 2026-02-18T06:47:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8551357440 bytes 2026-02-18T06:47:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8570689536 bytes 2026-02-18T06:47:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:47:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:47:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_10 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:47:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:47:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_10 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2026-02-18T06:47:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:47:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-18T06:47:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:47:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_10': 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12f4a7baf084 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12f4a7baf084 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:34 2026 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:34 2026 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:34 2026 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c38dfa61-f880-425c-89ec-99dda99f916d 2026-02-18T06:47:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-18T06:47:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:47:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=8 2026-02-18T06:47:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 37124096 bytes 2026-02-18T06:47:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 122598400 bytes 2026-02-18T06:47:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 172426240 bytes 2026-02-18T06:47:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 173033472 bytes 2026-02-18T06:47:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 225082368 bytes 2026-02-18T06:47:35.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 292086784 bytes 2026-02-18T06:47:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 300519424 bytes 2026-02-18T06:47:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 343161856 bytes 2026-02-18T06:47:35.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 369076224 bytes 2026-02-18T06:47:35.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 407026688 bytes 2026-02-18T06:47:35.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 518798336 bytes 2026-02-18T06:47:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 592615424 bytes 2026-02-18T06:47:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 598756352 bytes 2026-02-18T06:47:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 620425216 bytes 2026-02-18T06:47:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 686524416 bytes 2026-02-18T06:47:36.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 758337536 bytes 2026-02-18T06:47:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 791045120 bytes 2026-02-18T06:47:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 848517120 bytes 2026-02-18T06:47:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 886842368 bytes 2026-02-18T06:47:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 935356416 bytes 2026-02-18T06:47:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 954066944 bytes 2026-02-18T06:47:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 976261120 bytes 2026-02-18T06:47:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1025934336 bytes 2026-02-18T06:47:36.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1044189184 bytes 2026-02-18T06:47:36.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1077923840 bytes 2026-02-18T06:47:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1106790400 bytes 2026-02-18T06:47:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1150666752 bytes 2026-02-18T06:47:36.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1182187520 bytes 2026-02-18T06:47:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1261837312 bytes 2026-02-18T06:47:36.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1291036672 bytes 2026-02-18T06:47:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1296618496 bytes 2026-02-18T06:47:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1330797568 bytes 2026-02-18T06:47:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1363171328 bytes 2026-02-18T06:47:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1395350528 bytes 2026-02-18T06:47:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1423880192 bytes 2026-02-18T06:47:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1521419264 bytes 2026-02-18T06:47:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1549644800 bytes 2026-02-18T06:47:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1550284800 bytes 2026-02-18T06:47:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1645045760 bytes 2026-02-18T06:47:37.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1658285056 bytes 2026-02-18T06:47:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1678071808 bytes 2026-02-18T06:47:37.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1716517888 bytes 2026-02-18T06:47:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1772450816 bytes 2026-02-18T06:47:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1808220160 bytes 2026-02-18T06:47:37.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1876575232 bytes 2026-02-18T06:47:37.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1899379712 bytes 2026-02-18T06:47:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1930808320 bytes 2026-02-18T06:47:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1995426816 bytes 2026-02-18T06:47:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2056286208 bytes 2026-02-18T06:47:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2079172608 bytes 2026-02-18T06:47:38.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2119995392 bytes 2026-02-18T06:47:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2167099392 bytes 2026-02-18T06:47:38.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2228338688 bytes 2026-02-18T06:47:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2229713920 bytes 2026-02-18T06:47:38.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2293427200 bytes 2026-02-18T06:47:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2328504320 bytes 2026-02-18T06:47:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2329108480 bytes 2026-02-18T06:47:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2368132096 bytes 2026-02-18T06:47:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2394609664 bytes 2026-02-18T06:47:38.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2435398656 bytes 2026-02-18T06:47:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2478796800 bytes 2026-02-18T06:47:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2518912000 bytes 2026-02-18T06:47:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2555304960 bytes 2026-02-18T06:47:38.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2600804352 bytes 2026-02-18T06:47:38.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2651491328 bytes 2026-02-18T06:47:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2697032704 bytes 2026-02-18T06:47:38.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2697638912 bytes 2026-02-18T06:47:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2745155584 bytes 2026-02-18T06:47:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2795275264 bytes 2026-02-18T06:47:39.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2842987520 bytes 2026-02-18T06:47:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2893553664 bytes 2026-02-18T06:47:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2921194496 bytes 2026-02-18T06:47:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2979870720 bytes 2026-02-18T06:47:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3041507328 bytes 2026-02-18T06:47:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3042136064 bytes 2026-02-18T06:47:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3111793664 bytes 2026-02-18T06:47:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3119893504 bytes 2026-02-18T06:47:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3151218688 bytes 2026-02-18T06:47:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3226027008 bytes 2026-02-18T06:47:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3288931328 bytes 2026-02-18T06:47:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3325474816 bytes 2026-02-18T06:47:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3366429696 bytes 2026-02-18T06:47:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3411243008 bytes 2026-02-18T06:47:39.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3459674112 bytes 2026-02-18T06:47:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3488086016 bytes 2026-02-18T06:47:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3552378880 bytes 2026-02-18T06:47:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3604740096 bytes 2026-02-18T06:47:40.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3670751232 bytes 2026-02-18T06:47:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3708690432 bytes 2026-02-18T06:47:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3712513024 bytes 2026-02-18T06:47:40.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3766823936 bytes 2026-02-18T06:47:40.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3804112896 bytes 2026-02-18T06:47:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3889080320 bytes 2026-02-18T06:47:40.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4018476032 bytes 2026-02-18T06:47:40.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4075935744 bytes 2026-02-18T06:47:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4168185856 bytes 2026-02-18T06:47:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4172475392 bytes 2026-02-18T06:47:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4213940224 bytes 2026-02-18T06:47:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4303617024 bytes 2026-02-18T06:47:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4388256768 bytes 2026-02-18T06:47:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4451742720 bytes 2026-02-18T06:47:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4485820416 bytes 2026-02-18T06:47:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4511112192 bytes 2026-02-18T06:47:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4535977984 bytes 2026-02-18T06:47:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4589480960 bytes 2026-02-18T06:47:41.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4642189312 bytes 2026-02-18T06:47:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4657494016 bytes 2026-02-18T06:47:41.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4670379008 bytes 2026-02-18T06:47:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4698110976 bytes 2026-02-18T06:47:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4740907008 bytes 2026-02-18T06:47:41.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4765015040 bytes 2026-02-18T06:47:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4766116864 bytes 2026-02-18T06:47:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4826881024 bytes 2026-02-18T06:47:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4900775936 bytes 2026-02-18T06:47:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4945552384 bytes 2026-02-18T06:47:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5028028416 bytes 2026-02-18T06:47:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5052558336 bytes 2026-02-18T06:47:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5061811200 bytes 2026-02-18T06:47:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5101601792 bytes 2026-02-18T06:47:42.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5143802880 bytes 2026-02-18T06:47:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5171713024 bytes 2026-02-18T06:47:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5201105920 bytes 2026-02-18T06:47:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5214438400 bytes 2026-02-18T06:47:42.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5268311040 bytes 2026-02-18T06:47:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5268930560 bytes 2026-02-18T06:47:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5290981376 bytes 2026-02-18T06:47:42.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5330717696 bytes 2026-02-18T06:47:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5355931648 bytes 2026-02-18T06:47:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5419830272 bytes 2026-02-18T06:47:42.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5480594432 bytes 2026-02-18T06:47:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5497562112 bytes 2026-02-18T06:47:42.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5553943552 bytes 2026-02-18T06:47:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5617729536 bytes 2026-02-18T06:47:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5657916416 bytes 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_11 2026-02-18T06:47:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_11 2026-02-18T06:47:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:47:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:47:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2026-02-18T06:47:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:47:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_11': 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1300dd139da6 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1300dd139da6 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:43 2026 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:43 2026 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:43 2026 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 4a7b60e8-59db-4352-8976-1ffb0ad77ed8 2026-02-18T06:47:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-18T06:47:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:47:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:47:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 661504 bytes 2026-02-18T06:47:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 15489024 bytes 2026-02-18T06:47:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 67511296 bytes 2026-02-18T06:47:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 139237376 bytes 2026-02-18T06:47:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 209755136 bytes 2026-02-18T06:47:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 259248128 bytes 2026-02-18T06:47:44.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 296821760 bytes 2026-02-18T06:47:44.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 356535296 bytes 2026-02-18T06:47:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 389725184 bytes 2026-02-18T06:47:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 452453376 bytes 2026-02-18T06:47:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 463136768 bytes 2026-02-18T06:47:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 509725696 bytes 2026-02-18T06:47:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 510358528 bytes 2026-02-18T06:47:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 631403520 bytes 2026-02-18T06:47:44.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 649986048 bytes 2026-02-18T06:47:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 739344384 bytes 2026-02-18T06:47:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 774933504 bytes 2026-02-18T06:47:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 821781504 bytes 2026-02-18T06:47:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 841795584 bytes 2026-02-18T06:47:45.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 924334080 bytes 2026-02-18T06:47:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 997851136 bytes 2026-02-18T06:47:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1041080320 bytes 2026-02-18T06:47:45.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1084840960 bytes 2026-02-18T06:47:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1174293504 bytes 2026-02-18T06:47:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1209784320 bytes 2026-02-18T06:47:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1255332864 bytes 2026-02-18T06:47:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1293546496 bytes 2026-02-18T06:47:45.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1339177984 bytes 2026-02-18T06:47:45.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1380059136 bytes 2026-02-18T06:47:45.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1380669440 bytes 2026-02-18T06:47:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1444763648 bytes 2026-02-18T06:47:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1486087168 bytes 2026-02-18T06:47:45.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1525313536 bytes 2026-02-18T06:47:45.994 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:47:45.993+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbd2ccf790 2026-02-18T06:47:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1530326016 bytes 2026-02-18T06:47:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1574062080 bytes 2026-02-18T06:47:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1605877760 bytes 2026-02-18T06:47:46.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1615034368 bytes 2026-02-18T06:47:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1633804288 bytes 2026-02-18T06:47:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1679953920 bytes 2026-02-18T06:47:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1744162816 bytes 2026-02-18T06:47:46.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1795078144 bytes 2026-02-18T06:47:46.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1833457664 bytes 2026-02-18T06:47:46.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1903077376 bytes 2026-02-18T06:47:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1927867392 bytes 2026-02-18T06:47:46.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2029714432 bytes 2026-02-18T06:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2067750912 bytes 2026-02-18T06:47:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2104449024 bytes 2026-02-18T06:47:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2127817728 bytes 2026-02-18T06:47:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2235606016 bytes 2026-02-18T06:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2273115136 bytes 2026-02-18T06:47:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2318508032 bytes 2026-02-18T06:47:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2352057344 bytes 2026-02-18T06:47:47.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2386353152 bytes 2026-02-18T06:47:47.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2427663360 bytes 2026-02-18T06:47:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2462783488 bytes 2026-02-18T06:47:47.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2504786944 bytes 2026-02-18T06:47:47.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2564516864 bytes 2026-02-18T06:47:47.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2585269248 bytes 2026-02-18T06:47:47.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2696383488 bytes 2026-02-18T06:47:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2719819776 bytes 2026-02-18T06:47:47.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2840082432 bytes 2026-02-18T06:47:47.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2885352448 bytes 2026-02-18T06:47:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 3003217920 bytes 2026-02-18T06:47:47.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3032586240 bytes 2026-02-18T06:47:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3074470912 bytes 2026-02-18T06:47:48.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3150671872 bytes 2026-02-18T06:47:48.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3180782592 bytes 2026-02-18T06:47:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3223313408 bytes 2026-02-18T06:47:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3240391680 bytes 2026-02-18T06:47:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3284004864 bytes 2026-02-18T06:47:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3343985664 bytes 2026-02-18T06:47:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3388065792 bytes 2026-02-18T06:47:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3432345600 bytes 2026-02-18T06:47:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3482271744 bytes 2026-02-18T06:47:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3529876480 bytes 2026-02-18T06:47:48.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3588673536 bytes 2026-02-18T06:47:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3612156928 bytes 2026-02-18T06:47:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3659876352 bytes 2026-02-18T06:47:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3700503552 bytes 2026-02-18T06:47:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3721421824 bytes 2026-02-18T06:47:48.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3764419584 bytes 2026-02-18T06:47:48.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3796465664 bytes 2026-02-18T06:47:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3816414208 bytes 2026-02-18T06:47:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3831997440 bytes 2026-02-18T06:47:49.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3861096448 bytes 2026-02-18T06:47:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3916796928 bytes 2026-02-18T06:47:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3987854336 bytes 2026-02-18T06:47:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4096026624 bytes 2026-02-18T06:47:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4198347776 bytes 2026-02-18T06:47:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4236071936 bytes 2026-02-18T06:47:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4271623168 bytes 2026-02-18T06:47:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4306688000 bytes 2026-02-18T06:47:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4387555328 bytes 2026-02-18T06:47:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4388191232 bytes 2026-02-18T06:47:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4499271680 bytes 2026-02-18T06:47:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4536621056 bytes 2026-02-18T06:47:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4592224256 bytes 2026-02-18T06:47:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4662699008 bytes 2026-02-18T06:47:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4663293952 bytes 2026-02-18T06:47:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4737886208 bytes 2026-02-18T06:47:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4833907712 bytes 2026-02-18T06:47:50.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4885479424 bytes 2026-02-18T06:47:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4893033472 bytes 2026-02-18T06:47:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4916208640 bytes 2026-02-18T06:47:50.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4961347584 bytes 2026-02-18T06:47:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 5028883456 bytes 2026-02-18T06:47:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5056699392 bytes 2026-02-18T06:47:50.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5095403520 bytes 2026-02-18T06:47:50.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5098395648 bytes 2026-02-18T06:47:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5128000512 bytes 2026-02-18T06:47:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5188294656 bytes 2026-02-18T06:47:50.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5226023936 bytes 2026-02-18T06:47:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5229542400 bytes 2026-02-18T06:47:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5284147200 bytes 2026-02-18T06:47:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5297165312 bytes 2026-02-18T06:47:50.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5328216064 bytes 2026-02-18T06:47:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5403008000 bytes 2026-02-18T06:47:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5405733888 bytes 2026-02-18T06:47:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5430809600 bytes 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_12 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:47:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_12 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:47:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2026-02-18T06:47:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:47:51.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2026-02-18T06:47:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_12': 2026-02-18T06:47:52.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 130ca359dcc1 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.130ca359dcc1 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:51 2026 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:51 2026 2026-02-18T06:47:52.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:51 2026 2026-02-18T06:47:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:47:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:47:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e98f19fc-6436-4e93-bab9-e22e39c2f301 2026-02-18T06:47:52.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:47:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:47:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-18T06:47:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:47:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:47:52.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:47:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:47:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=10 2026-02-18T06:47:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:47:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:47:52.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 89195520 bytes 2026-02-18T06:47:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 158876672 bytes 2026-02-18T06:47:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 210514944 bytes 2026-02-18T06:47:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 248883200 bytes 2026-02-18T06:47:52.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 322036736 bytes 2026-02-18T06:47:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 331174912 bytes 2026-02-18T06:47:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 351465472 bytes 2026-02-18T06:47:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 453529600 bytes 2026-02-18T06:47:52.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 502423552 bytes 2026-02-18T06:47:52.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 529989632 bytes 2026-02-18T06:47:52.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 540918784 bytes 2026-02-18T06:47:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 610289664 bytes 2026-02-18T06:47:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 615858176 bytes 2026-02-18T06:47:52.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 701241344 bytes 2026-02-18T06:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 720381952 bytes 2026-02-18T06:47:53.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 812747776 bytes 2026-02-18T06:47:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 826300416 bytes 2026-02-18T06:47:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 884017152 bytes 2026-02-18T06:47:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 919454720 bytes 2026-02-18T06:47:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1003527168 bytes 2026-02-18T06:47:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1023332352 bytes 2026-02-18T06:47:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1106792448 bytes 2026-02-18T06:47:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1167127552 bytes 2026-02-18T06:47:53.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1198948352 bytes 2026-02-18T06:47:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1257743360 bytes 2026-02-18T06:47:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1269222400 bytes 2026-02-18T06:47:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1302661120 bytes 2026-02-18T06:47:53.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1330825216 bytes 2026-02-18T06:47:53.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1331431424 bytes 2026-02-18T06:47:53.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1374013440 bytes 2026-02-18T06:47:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1374608384 bytes 2026-02-18T06:47:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1480152064 bytes 2026-02-18T06:47:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1498771456 bytes 2026-02-18T06:47:54.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1499375616 bytes 2026-02-18T06:47:54.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1522736128 bytes 2026-02-18T06:47:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1571085312 bytes 2026-02-18T06:47:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1633781760 bytes 2026-02-18T06:47:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1665487872 bytes 2026-02-18T06:47:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1705140224 bytes 2026-02-18T06:47:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1775678464 bytes 2026-02-18T06:47:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1811617792 bytes 2026-02-18T06:47:54.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1887086592 bytes 2026-02-18T06:47:54.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1941663744 bytes 2026-02-18T06:47:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1942277120 bytes 2026-02-18T06:47:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1954549760 bytes 2026-02-18T06:47:54.993 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.509105881279312) > weight (0.25) 2026-02-18T06:47:54.993 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5955995872473364) > weight (0.25) 2026-02-18T06:47:54.993 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T06:47:54.993 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T06:47:54.993 INFO:tasks.rbd_mirror_thrash:waiting for 1.6202345252979744 secs before reviving daemons 2026-02-18T06:47:54.994 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:47:54.993+0000 7f3ff82b7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 29929) UID: 1000 2026-02-18T06:47:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1997229056 bytes 2026-02-18T06:47:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2040757248 bytes 2026-02-18T06:47:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2083308544 bytes 2026-02-18T06:47:55.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2157993984 bytes 2026-02-18T06:47:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2184970240 bytes 2026-02-18T06:47:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2236883968 bytes 2026-02-18T06:47:55.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2287790080 bytes 2026-02-18T06:47:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2333652992 bytes 2026-02-18T06:47:55.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2406382592 bytes 2026-02-18T06:47:55.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2422696960 bytes 2026-02-18T06:47:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2489722880 bytes 2026-02-18T06:47:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2536551424 bytes 2026-02-18T06:47:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2543735808 bytes 2026-02-18T06:47:55.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2571253760 bytes 2026-02-18T06:47:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2675610624 bytes 2026-02-18T06:47:56.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2777057280 bytes 2026-02-18T06:47:56.074 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T06:47:56.072+0000 7f8353d0d640 -1 rbd::mirror::ImageReplayer: 0x56127efa0000 [3/e6c1e58d-7558-4bf6-9061-94218ae350a1] start: already running 2026-02-18T06:47:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2810743808 bytes 2026-02-18T06:47:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2819894272 bytes 2026-02-18T06:47:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2851390464 bytes 2026-02-18T06:47:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2891567104 bytes 2026-02-18T06:47:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2946685952 bytes 2026-02-18T06:47:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3002605568 bytes 2026-02-18T06:47:56.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3029126144 bytes 2026-02-18T06:47:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3141170176 bytes 2026-02-18T06:47:56.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3205014528 bytes 2026-02-18T06:47:56.614 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T06:47:56.614 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:47:56.615 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T06:47:56.615 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T06:47:56.615 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T06:47:56.616 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T06:47:56.617 INFO:tasks.rbd_mirror_thrash:waiting for 78.43707789587958 secs before thrashing 2026-02-18T06:47:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3245016064 bytes 2026-02-18T06:47:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3308154880 bytes 2026-02-18T06:47:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3310110720 bytes 2026-02-18T06:47:56.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3320148992 bytes 2026-02-18T06:47:56.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3320763392 bytes 2026-02-18T06:47:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3354295296 bytes 2026-02-18T06:47:56.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3400540160 bytes 2026-02-18T06:47:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3446566912 bytes 2026-02-18T06:47:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3471628288 bytes 2026-02-18T06:47:57.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3535403008 bytes 2026-02-18T06:47:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3610401792 bytes 2026-02-18T06:47:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3702046720 bytes 2026-02-18T06:47:57.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3723756544 bytes 2026-02-18T06:47:57.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3787836416 bytes 2026-02-18T06:47:57.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3844519936 bytes 2026-02-18T06:47:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3919930368 bytes 2026-02-18T06:47:57.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3949842432 bytes 2026-02-18T06:47:57.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3985878016 bytes 2026-02-18T06:47:57.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3998213120 bytes 2026-02-18T06:47:57.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4044616704 bytes 2026-02-18T06:47:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4117489664 bytes 2026-02-18T06:47:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4143423488 bytes 2026-02-18T06:47:57.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4144027648 bytes 2026-02-18T06:47:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4175575040 bytes 2026-02-18T06:47:58.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4234702848 bytes 2026-02-18T06:47:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4241373184 bytes 2026-02-18T06:47:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4242038784 bytes 2026-02-18T06:47:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4365660160 bytes 2026-02-18T06:47:58.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4405588992 bytes 2026-02-18T06:47:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4422827008 bytes 2026-02-18T06:47:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4425720832 bytes 2026-02-18T06:47:58.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4426316800 bytes 2026-02-18T06:47:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4444987392 bytes 2026-02-18T06:47:58.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4485954560 bytes 2026-02-18T06:47:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4524832768 bytes 2026-02-18T06:47:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4638955520 bytes 2026-02-18T06:47:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4674137088 bytes 2026-02-18T06:47:58.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4683375616 bytes 2026-02-18T06:47:58.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4716131328 bytes 2026-02-18T06:47:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4804700160 bytes 2026-02-18T06:47:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4805312512 bytes 2026-02-18T06:47:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4905516032 bytes 2026-02-18T06:47:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4936306688 bytes 2026-02-18T06:47:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4997551104 bytes 2026-02-18T06:47:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5068440576 bytes 2026-02-18T06:47:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5154385920 bytes 2026-02-18T06:47:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5165800448 bytes 2026-02-18T06:47:59.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5201010688 bytes 2026-02-18T06:47:59.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5228437504 bytes 2026-02-18T06:47:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5321888768 bytes 2026-02-18T06:47:59.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5331500032 bytes 2026-02-18T06:47:59.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5369096192 bytes 2026-02-18T06:47:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5408754688 bytes 2026-02-18T06:47:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5449938944 bytes 2026-02-18T06:47:59.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5468637184 bytes 2026-02-18T06:47:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5525932032 bytes 2026-02-18T06:47:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5579943936 bytes 2026-02-18T06:47:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5642208256 bytes 2026-02-18T06:48:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5693917184 bytes 2026-02-18T06:48:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5744742400 bytes 2026-02-18T06:48:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5770341376 bytes 2026-02-18T06:48:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5854586880 bytes 2026-02-18T06:48:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5871364096 bytes 2026-02-18T06:48:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5950927872 bytes 2026-02-18T06:48:00.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5990880256 bytes 2026-02-18T06:48:00.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5993190400 bytes 2026-02-18T06:48:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6035191808 bytes 2026-02-18T06:48:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6046508032 bytes 2026-02-18T06:48:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6103177216 bytes 2026-02-18T06:48:00.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6129106944 bytes 2026-02-18T06:48:00.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6212883456 bytes 2026-02-18T06:48:00.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6232741888 bytes 2026-02-18T06:48:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6290634752 bytes 2026-02-18T06:48:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6426211328 bytes 2026-02-18T06:48:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6470912000 bytes 2026-02-18T06:48:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6538763264 bytes 2026-02-18T06:48:01.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6556101632 bytes 2026-02-18T06:48:01.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6614618112 bytes 2026-02-18T06:48:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6688777216 bytes 2026-02-18T06:48:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6719048704 bytes 2026-02-18T06:48:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6739863552 bytes 2026-02-18T06:48:01.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6776665088 bytes 2026-02-18T06:48:01.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6870991872 bytes 2026-02-18T06:48:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6907299840 bytes 2026-02-18T06:48:01.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6921556992 bytes 2026-02-18T06:48:01.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6968589312 bytes 2026-02-18T06:48:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7024247808 bytes 2026-02-18T06:48:01.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7024867328 bytes 2026-02-18T06:48:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7098390528 bytes 2026-02-18T06:48:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7115299840 bytes 2026-02-18T06:48:01.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7140629504 bytes 2026-02-18T06:48:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_13 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_13 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2026-02-18T06:48:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:03.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2026-02-18T06:48:03.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_13': 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 131e95f1a05e 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.131e95f1a05e 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:02 2026 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:02 2026 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:02 2026 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: ac0bdee8-83d2-49fa-8584-10a7ba48c5e6 2026-02-18T06:48:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:48:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:48:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:03.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:03.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 62244864 bytes 2026-02-18T06:48:03.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 148057088 bytes 2026-02-18T06:48:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 183135232 bytes 2026-02-18T06:48:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 282003456 bytes 2026-02-18T06:48:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 306820096 bytes 2026-02-18T06:48:03.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 378244096 bytes 2026-02-18T06:48:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 378882048 bytes 2026-02-18T06:48:03.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 403025920 bytes 2026-02-18T06:48:03.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 463184896 bytes 2026-02-18T06:48:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 549114880 bytes 2026-02-18T06:48:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 586914816 bytes 2026-02-18T06:48:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 671252480 bytes 2026-02-18T06:48:03.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 697709568 bytes 2026-02-18T06:48:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 766988288 bytes 2026-02-18T06:48:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 844873728 bytes 2026-02-18T06:48:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 860470272 bytes 2026-02-18T06:48:04.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 902473728 bytes 2026-02-18T06:48:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 959399936 bytes 2026-02-18T06:48:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 1008862208 bytes 2026-02-18T06:48:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1049936896 bytes 2026-02-18T06:48:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1105359872 bytes 2026-02-18T06:48:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1105996800 bytes 2026-02-18T06:48:04.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1186824192 bytes 2026-02-18T06:48:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1260728320 bytes 2026-02-18T06:48:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1303502848 bytes 2026-02-18T06:48:04.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1340257280 bytes 2026-02-18T06:48:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1362042880 bytes 2026-02-18T06:48:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1479525376 bytes 2026-02-18T06:48:05.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1555205120 bytes 2026-02-18T06:48:05.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1640629248 bytes 2026-02-18T06:48:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1690689536 bytes 2026-02-18T06:48:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1696363520 bytes 2026-02-18T06:48:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1746211840 bytes 2026-02-18T06:48:05.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1750917120 bytes 2026-02-18T06:48:05.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1791742976 bytes 2026-02-18T06:48:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1810066432 bytes 2026-02-18T06:48:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1863225344 bytes 2026-02-18T06:48:05.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1934908416 bytes 2026-02-18T06:48:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1978300416 bytes 2026-02-18T06:48:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2099272704 bytes 2026-02-18T06:48:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2112178176 bytes 2026-02-18T06:48:05.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2129333248 bytes 2026-02-18T06:48:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2222469120 bytes 2026-02-18T06:48:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2315406336 bytes 2026-02-18T06:48:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2350709760 bytes 2026-02-18T06:48:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2408413184 bytes 2026-02-18T06:48:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2429358080 bytes 2026-02-18T06:48:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2463128576 bytes 2026-02-18T06:48:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2485881856 bytes 2026-02-18T06:48:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2576147456 bytes 2026-02-18T06:48:06.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2605918208 bytes 2026-02-18T06:48:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2644265984 bytes 2026-02-18T06:48:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2649873408 bytes 2026-02-18T06:48:06.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2681239552 bytes 2026-02-18T06:48:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2720005120 bytes 2026-02-18T06:48:06.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2735377408 bytes 2026-02-18T06:48:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2772770816 bytes 2026-02-18T06:48:06.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2797462528 bytes 2026-02-18T06:48:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2887919616 bytes 2026-02-18T06:48:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2938947584 bytes 2026-02-18T06:48:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2943086592 bytes 2026-02-18T06:48:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2957538304 bytes 2026-02-18T06:48:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 3010260992 bytes 2026-02-18T06:48:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3044446208 bytes 2026-02-18T06:48:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3065120768 bytes 2026-02-18T06:48:07.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3135241216 bytes 2026-02-18T06:48:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3190099968 bytes 2026-02-18T06:48:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3231191040 bytes 2026-02-18T06:48:07.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3352087552 bytes 2026-02-18T06:48:07.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3379814400 bytes 2026-02-18T06:48:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3422190592 bytes 2026-02-18T06:48:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3498529792 bytes 2026-02-18T06:48:07.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3552652288 bytes 2026-02-18T06:48:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3594986496 bytes 2026-02-18T06:48:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3619382272 bytes 2026-02-18T06:48:08.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3714583552 bytes 2026-02-18T06:48:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3777843200 bytes 2026-02-18T06:48:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3837690880 bytes 2026-02-18T06:48:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3838298112 bytes 2026-02-18T06:48:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3864265728 bytes 2026-02-18T06:48:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3896939520 bytes 2026-02-18T06:48:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3914184704 bytes 2026-02-18T06:48:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3919819776 bytes 2026-02-18T06:48:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3951564800 bytes 2026-02-18T06:48:08.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3984561152 bytes 2026-02-18T06:48:08.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4073447424 bytes 2026-02-18T06:48:08.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4120979456 bytes 2026-02-18T06:48:08.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4164867072 bytes 2026-02-18T06:48:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4219133952 bytes 2026-02-18T06:48:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4264844288 bytes 2026-02-18T06:48:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4313008128 bytes 2026-02-18T06:48:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4332131328 bytes 2026-02-18T06:48:09.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4364903424 bytes 2026-02-18T06:48:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4407354368 bytes 2026-02-18T06:48:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4445061120 bytes 2026-02-18T06:48:09.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4447040512 bytes 2026-02-18T06:48:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4476811264 bytes 2026-02-18T06:48:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4485490688 bytes 2026-02-18T06:48:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4524807168 bytes 2026-02-18T06:48:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4546169856 bytes 2026-02-18T06:48:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4615759872 bytes 2026-02-18T06:48:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4743554048 bytes 2026-02-18T06:48:09.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4773308416 bytes 2026-02-18T06:48:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4825470976 bytes 2026-02-18T06:48:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4873622528 bytes 2026-02-18T06:48:09.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4922193920 bytes 2026-02-18T06:48:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4969172992 bytes 2026-02-18T06:48:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_14 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_14 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:48:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2026-02-18T06:48:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2026-02-18T06:48:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_14': 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 132a648b4fa0 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.132a648b4fa0 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:10 2026 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:10 2026 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:10 2026 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 28c18533-c7df-40a3-9c10-0a9ba74d304a 2026-02-18T06:48:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-18T06:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:11.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 10938368 bytes 2026-02-18T06:48:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 36760576 bytes 2026-02-18T06:48:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 42278912 bytes 2026-02-18T06:48:11.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 116805632 bytes 2026-02-18T06:48:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 157982720 bytes 2026-02-18T06:48:11.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 206848000 bytes 2026-02-18T06:48:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 255619072 bytes 2026-02-18T06:48:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 283092992 bytes 2026-02-18T06:48:11.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 283700224 bytes 2026-02-18T06:48:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 293200896 bytes 2026-02-18T06:48:11.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 351509504 bytes 2026-02-18T06:48:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 354244608 bytes 2026-02-18T06:48:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 367345664 bytes 2026-02-18T06:48:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 388079616 bytes 2026-02-18T06:48:11.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 389068800 bytes 2026-02-18T06:48:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 429979648 bytes 2026-02-18T06:48:12.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 473250816 bytes 2026-02-18T06:48:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 528345088 bytes 2026-02-18T06:48:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 591504384 bytes 2026-02-18T06:48:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 617153536 bytes 2026-02-18T06:48:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 639845376 bytes 2026-02-18T06:48:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 673771520 bytes 2026-02-18T06:48:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 702024704 bytes 2026-02-18T06:48:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 764940288 bytes 2026-02-18T06:48:12.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 788911104 bytes 2026-02-18T06:48:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 804840448 bytes 2026-02-18T06:48:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 845455360 bytes 2026-02-18T06:48:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 899162112 bytes 2026-02-18T06:48:12.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 945054720 bytes 2026-02-18T06:48:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 967049216 bytes 2026-02-18T06:48:12.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1014312960 bytes 2026-02-18T06:48:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1053429760 bytes 2026-02-18T06:48:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1065217024 bytes 2026-02-18T06:48:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1109350400 bytes 2026-02-18T06:48:13.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1191056384 bytes 2026-02-18T06:48:13.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1245926400 bytes 2026-02-18T06:48:13.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1278631936 bytes 2026-02-18T06:48:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1331966976 bytes 2026-02-18T06:48:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1357200384 bytes 2026-02-18T06:48:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1372388352 bytes 2026-02-18T06:48:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1441219584 bytes 2026-02-18T06:48:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1489921024 bytes 2026-02-18T06:48:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1505891328 bytes 2026-02-18T06:48:13.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1548003328 bytes 2026-02-18T06:48:13.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1630113792 bytes 2026-02-18T06:48:13.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1639763968 bytes 2026-02-18T06:48:13.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1707352064 bytes 2026-02-18T06:48:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1723229184 bytes 2026-02-18T06:48:13.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1764827136 bytes 2026-02-18T06:48:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1766085632 bytes 2026-02-18T06:48:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1766711296 bytes 2026-02-18T06:48:14.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1859459072 bytes 2026-02-18T06:48:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1930570752 bytes 2026-02-18T06:48:14.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2007843840 bytes 2026-02-18T06:48:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2079066112 bytes 2026-02-18T06:48:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2100227072 bytes 2026-02-18T06:48:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2129756160 bytes 2026-02-18T06:48:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2149889024 bytes 2026-02-18T06:48:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2168895488 bytes 2026-02-18T06:48:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2209207296 bytes 2026-02-18T06:48:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2285240320 bytes 2026-02-18T06:48:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2300256256 bytes 2026-02-18T06:48:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2319002624 bytes 2026-02-18T06:48:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2319592448 bytes 2026-02-18T06:48:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2334354432 bytes 2026-02-18T06:48:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2416526336 bytes 2026-02-18T06:48:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2418562048 bytes 2026-02-18T06:48:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2439368704 bytes 2026-02-18T06:48:15.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2484031488 bytes 2026-02-18T06:48:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2525660160 bytes 2026-02-18T06:48:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2528871424 bytes 2026-02-18T06:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2573850624 bytes 2026-02-18T06:48:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2638183424 bytes 2026-02-18T06:48:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2661999616 bytes 2026-02-18T06:48:15.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2690501632 bytes 2026-02-18T06:48:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2704672768 bytes 2026-02-18T06:48:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2774508544 bytes 2026-02-18T06:48:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2861927424 bytes 2026-02-18T06:48:15.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2909679616 bytes 2026-02-18T06:48:15.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2912891904 bytes 2026-02-18T06:48:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 2943309824 bytes 2026-02-18T06:48:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 2982218752 bytes 2026-02-18T06:48:15.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3020817408 bytes 2026-02-18T06:48:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3060347904 bytes 2026-02-18T06:48:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3076002816 bytes 2026-02-18T06:48:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3098580992 bytes 2026-02-18T06:48:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3131143168 bytes 2026-02-18T06:48:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3215511552 bytes 2026-02-18T06:48:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3291242496 bytes 2026-02-18T06:48:16.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3352119296 bytes 2026-02-18T06:48:16.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3372672000 bytes 2026-02-18T06:48:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3439305728 bytes 2026-02-18T06:48:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3472738304 bytes 2026-02-18T06:48:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3494264832 bytes 2026-02-18T06:48:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3535031296 bytes 2026-02-18T06:48:16.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3557666816 bytes 2026-02-18T06:48:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3565966336 bytes 2026-02-18T06:48:16.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3576090624 bytes 2026-02-18T06:48:16.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3628044288 bytes 2026-02-18T06:48:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3635486720 bytes 2026-02-18T06:48:16.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3697522688 bytes 2026-02-18T06:48:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3711126528 bytes 2026-02-18T06:48:16.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3738391552 bytes 2026-02-18T06:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 3785964544 bytes 2026-02-18T06:48:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 3786577920 bytes 2026-02-18T06:48:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 3829566464 bytes 2026-02-18T06:48:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 3844511744 bytes 2026-02-18T06:48:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 3851764736 bytes 2026-02-18T06:48:17.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 3892148224 bytes 2026-02-18T06:48:17.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 3949725696 bytes 2026-02-18T06:48:17.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 3977530368 bytes 2026-02-18T06:48:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 3983212544 bytes 2026-02-18T06:48:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 3996414976 bytes 2026-02-18T06:48:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4029841408 bytes 2026-02-18T06:48:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4102690816 bytes 2026-02-18T06:48:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4198463488 bytes 2026-02-18T06:48:17.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4236826624 bytes 2026-02-18T06:48:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4278647808 bytes 2026-02-18T06:48:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4348092416 bytes 2026-02-18T06:48:17.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4360582144 bytes 2026-02-18T06:48:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4384174080 bytes 2026-02-18T06:48:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4438340608 bytes 2026-02-18T06:48:18.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4478888960 bytes 2026-02-18T06:48:18.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4509227008 bytes 2026-02-18T06:48:18.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4518267904 bytes 2026-02-18T06:48:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 4533472256 bytes 2026-02-18T06:48:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 4575712256 bytes 2026-02-18T06:48:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 4604968960 bytes 2026-02-18T06:48:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 4612286464 bytes 2026-02-18T06:48:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 4676356096 bytes 2026-02-18T06:48:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 4698231808 bytes 2026-02-18T06:48:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 4718453760 bytes 2026-02-18T06:48:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 4731311104 bytes 2026-02-18T06:48:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 4801780736 bytes 2026-02-18T06:48:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 4851338240 bytes 2026-02-18T06:48:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 4877125632 bytes 2026-02-18T06:48:18.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 4878940160 bytes 2026-02-18T06:48:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 4903801856 bytes 2026-02-18T06:48:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 4960856064 bytes 2026-02-18T06:48:18.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5039717376 bytes 2026-02-18T06:48:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5104271360 bytes 2026-02-18T06:48:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5164913664 bytes 2026-02-18T06:48:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5183507456 bytes 2026-02-18T06:48:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5208782848 bytes 2026-02-18T06:48:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5306391552 bytes 2026-02-18T06:48:19.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5363237888 bytes 2026-02-18T06:48:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5459372032 bytes 2026-02-18T06:48:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5554326528 bytes 2026-02-18T06:48:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5575249920 bytes 2026-02-18T06:48:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 5610502144 bytes 2026-02-18T06:48:19.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 5635942400 bytes 2026-02-18T06:48:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 5660889088 bytes 2026-02-18T06:48:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 5670688768 bytes 2026-02-18T06:48:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 5674055680 bytes 2026-02-18T06:48:19.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 5703577600 bytes 2026-02-18T06:48:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 5727353856 bytes 2026-02-18T06:48:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 5792378880 bytes 2026-02-18T06:48:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 5816436736 bytes 2026-02-18T06:48:20.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 5851142144 bytes 2026-02-18T06:48:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 5926223872 bytes 2026-02-18T06:48:20.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 5941480448 bytes 2026-02-18T06:48:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 5988493312 bytes 2026-02-18T06:48:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6025155584 bytes 2026-02-18T06:48:20.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6068765696 bytes 2026-02-18T06:48:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6069382144 bytes 2026-02-18T06:48:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6106742784 bytes 2026-02-18T06:48:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6142154752 bytes 2026-02-18T06:48:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6199924736 bytes 2026-02-18T06:48:20.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6232875008 bytes 2026-02-18T06:48:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6312879104 bytes 2026-02-18T06:48:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6330725376 bytes 2026-02-18T06:48:20.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6362368000 bytes 2026-02-18T06:48:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6384691200 bytes 2026-02-18T06:48:20.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6448602112 bytes 2026-02-18T06:48:21.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 6543116288 bytes 2026-02-18T06:48:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 6618781696 bytes 2026-02-18T06:48:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 6662468608 bytes 2026-02-18T06:48:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 6729563136 bytes 2026-02-18T06:48:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 6741982208 bytes 2026-02-18T06:48:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 6781250560 bytes 2026-02-18T06:48:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 6826010624 bytes 2026-02-18T06:48:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 6848026624 bytes 2026-02-18T06:48:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 6877260800 bytes 2026-02-18T06:48:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 6951063552 bytes 2026-02-18T06:48:21.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7028636672 bytes 2026-02-18T06:48:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7092631552 bytes 2026-02-18T06:48:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7157606400 bytes 2026-02-18T06:48:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7210289152 bytes 2026-02-18T06:48:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7238384640 bytes 2026-02-18T06:48:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7321402368 bytes 2026-02-18T06:48:22.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7370447872 bytes 2026-02-18T06:48:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7371069440 bytes 2026-02-18T06:48:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7436824576 bytes 2026-02-18T06:48:22.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7463189504 bytes 2026-02-18T06:48:22.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7527005184 bytes 2026-02-18T06:48:22.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7553698816 bytes 2026-02-18T06:48:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7565965312 bytes 2026-02-18T06:48:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 7647405056 bytes 2026-02-18T06:48:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 7699863552 bytes 2026-02-18T06:48:22.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 7763814400 bytes 2026-02-18T06:48:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 7781688320 bytes 2026-02-18T06:48:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 7841328128 bytes 2026-02-18T06:48:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 7884388352 bytes 2026-02-18T06:48:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 7925171200 bytes 2026-02-18T06:48:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 7925805056 bytes 2026-02-18T06:48:22.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 7954326528 bytes 2026-02-18T06:48:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8007927808 bytes 2026-02-18T06:48:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8026401792 bytes 2026-02-18T06:48:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8026993664 bytes 2026-02-18T06:48:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8094962688 bytes 2026-02-18T06:48:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8179569664 bytes 2026-02-18T06:48:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8318729216 bytes 2026-02-18T06:48:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8357089280 bytes 2026-02-18T06:48:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 8417163264 bytes 2026-02-18T06:48:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 8496324608 bytes 2026-02-18T06:48:23.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 8528706560 bytes 2026-02-18T06:48:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 8599949312 bytes 2026-02-18T06:48:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 8623307776 bytes 2026-02-18T06:48:24.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 8664044544 bytes 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_15 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_15 2026-02-18T06:48:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2026-02-18T06:48:24.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:24.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_15': 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1336b655463b 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1336b655463b 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:24 2026 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:24 2026 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:24 2026 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 35047dfb-3412-4601-ba30-18dbf9de2aaf 2026-02-18T06:48:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:48:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:48:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:25.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 132573184 bytes 2026-02-18T06:48:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 140703744 bytes 2026-02-18T06:48:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 167341056 bytes 2026-02-18T06:48:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 169850880 bytes 2026-02-18T06:48:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 188062720 bytes 2026-02-18T06:48:25.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 196966400 bytes 2026-02-18T06:48:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 217025536 bytes 2026-02-18T06:48:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 295636992 bytes 2026-02-18T06:48:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 351361024 bytes 2026-02-18T06:48:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 390082560 bytes 2026-02-18T06:48:25.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 439231488 bytes 2026-02-18T06:48:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 441870336 bytes 2026-02-18T06:48:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 537006080 bytes 2026-02-18T06:48:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 650942464 bytes 2026-02-18T06:48:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 704167936 bytes 2026-02-18T06:48:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 803921920 bytes 2026-02-18T06:48:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 851016704 bytes 2026-02-18T06:48:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 916560896 bytes 2026-02-18T06:48:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 1001296896 bytes 2026-02-18T06:48:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1078865920 bytes 2026-02-18T06:48:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1137410048 bytes 2026-02-18T06:48:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1160851456 bytes 2026-02-18T06:48:26.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1212227584 bytes 2026-02-18T06:48:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1303465984 bytes 2026-02-18T06:48:26.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1324547072 bytes 2026-02-18T06:48:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1395487744 bytes 2026-02-18T06:48:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1430479872 bytes 2026-02-18T06:48:27.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1431112704 bytes 2026-02-18T06:48:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1521128448 bytes 2026-02-18T06:48:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1540176896 bytes 2026-02-18T06:48:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1592095744 bytes 2026-02-18T06:48:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1620281344 bytes 2026-02-18T06:48:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1707563008 bytes 2026-02-18T06:48:27.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1744429056 bytes 2026-02-18T06:48:27.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1768018944 bytes 2026-02-18T06:48:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1852112896 bytes 2026-02-18T06:48:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1876614144 bytes 2026-02-18T06:48:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1902793728 bytes 2026-02-18T06:48:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1964516352 bytes 2026-02-18T06:48:27.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1992441856 bytes 2026-02-18T06:48:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2050613248 bytes 2026-02-18T06:48:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2102836224 bytes 2026-02-18T06:48:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2123943936 bytes 2026-02-18T06:48:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2178451456 bytes 2026-02-18T06:48:28.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2196348928 bytes 2026-02-18T06:48:28.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2215531520 bytes 2026-02-18T06:48:28.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2244985856 bytes 2026-02-18T06:48:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2287998976 bytes 2026-02-18T06:48:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2301190144 bytes 2026-02-18T06:48:28.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2332720128 bytes 2026-02-18T06:48:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2379384832 bytes 2026-02-18T06:48:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2426406912 bytes 2026-02-18T06:48:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2470895616 bytes 2026-02-18T06:48:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2489322496 bytes 2026-02-18T06:48:28.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2532926464 bytes 2026-02-18T06:48:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2586295296 bytes 2026-02-18T06:48:28.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2591129600 bytes 2026-02-18T06:48:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2609988608 bytes 2026-02-18T06:48:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2655196160 bytes 2026-02-18T06:48:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2675744768 bytes 2026-02-18T06:48:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2676357120 bytes 2026-02-18T06:48:29.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2746340352 bytes 2026-02-18T06:48:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2772497408 bytes 2026-02-18T06:48:29.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2813314048 bytes 2026-02-18T06:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2841039872 bytes 2026-02-18T06:48:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2879171584 bytes 2026-02-18T06:48:29.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2921384960 bytes 2026-02-18T06:48:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2947591168 bytes 2026-02-18T06:48:29.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2952382464 bytes 2026-02-18T06:48:29.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3043913728 bytes 2026-02-18T06:48:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3132822528 bytes 2026-02-18T06:48:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3252247552 bytes 2026-02-18T06:48:29.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3295009792 bytes 2026-02-18T06:48:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3334339584 bytes 2026-02-18T06:48:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3377463296 bytes 2026-02-18T06:48:30.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3428724736 bytes 2026-02-18T06:48:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3448673280 bytes 2026-02-18T06:48:30.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3466363904 bytes 2026-02-18T06:48:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3484546048 bytes 2026-02-18T06:48:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3580489728 bytes 2026-02-18T06:48:30.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3660792832 bytes 2026-02-18T06:48:30.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3701784576 bytes 2026-02-18T06:48:30.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3775995904 bytes 2026-02-18T06:48:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3829272576 bytes 2026-02-18T06:48:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3859516416 bytes 2026-02-18T06:48:30.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3888247808 bytes 2026-02-18T06:48:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3966804992 bytes 2026-02-18T06:48:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4013497344 bytes 2026-02-18T06:48:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4048813056 bytes 2026-02-18T06:48:31.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4094730240 bytes 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_16 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:31.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_16 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2026-02-18T06:48:31.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:31.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2026-02-18T06:48:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_16': 2026-02-18T06:48:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1342c8eca257 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1342c8eca257 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:31 2026 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:31 2026 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:31 2026 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c8eb0f42-d61b-4ba7-ad8e-859cc232365a 2026-02-18T06:48:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-18T06:48:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:48:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:48:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 36845568 bytes 2026-02-18T06:48:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 42269696 bytes 2026-02-18T06:48:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 103236608 bytes 2026-02-18T06:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 189556736 bytes 2026-02-18T06:48:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 194992128 bytes 2026-02-18T06:48:32.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 197107712 bytes 2026-02-18T06:48:32.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 309397504 bytes 2026-02-18T06:48:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 347242496 bytes 2026-02-18T06:48:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 406296576 bytes 2026-02-18T06:48:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 431652864 bytes 2026-02-18T06:48:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 553861120 bytes 2026-02-18T06:48:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 652008448 bytes 2026-02-18T06:48:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 653144064 bytes 2026-02-18T06:48:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 687220736 bytes 2026-02-18T06:48:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 726771712 bytes 2026-02-18T06:48:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 782811136 bytes 2026-02-18T06:48:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 844539904 bytes 2026-02-18T06:48:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 893771776 bytes 2026-02-18T06:48:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 931834880 bytes 2026-02-18T06:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 984497152 bytes 2026-02-18T06:48:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1006939136 bytes 2026-02-18T06:48:33.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1029396480 bytes 2026-02-18T06:48:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1053227008 bytes 2026-02-18T06:48:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1116467200 bytes 2026-02-18T06:48:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1135256576 bytes 2026-02-18T06:48:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1199151104 bytes 2026-02-18T06:48:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1317217280 bytes 2026-02-18T06:48:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1352416256 bytes 2026-02-18T06:48:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1401627648 bytes 2026-02-18T06:48:34.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1434204160 bytes 2026-02-18T06:48:34.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1458342912 bytes 2026-02-18T06:48:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1460296704 bytes 2026-02-18T06:48:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1539022848 bytes 2026-02-18T06:48:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1578012672 bytes 2026-02-18T06:48:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1709905920 bytes 2026-02-18T06:48:34.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1767776256 bytes 2026-02-18T06:48:34.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1768414208 bytes 2026-02-18T06:48:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1774655488 bytes 2026-02-18T06:48:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1873579008 bytes 2026-02-18T06:48:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1893351424 bytes 2026-02-18T06:48:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1919335424 bytes 2026-02-18T06:48:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1963046912 bytes 2026-02-18T06:48:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2016689152 bytes 2026-02-18T06:48:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2084065280 bytes 2026-02-18T06:48:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2143240192 bytes 2026-02-18T06:48:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2183468032 bytes 2026-02-18T06:48:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2266529792 bytes 2026-02-18T06:48:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2279775232 bytes 2026-02-18T06:48:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2297747456 bytes 2026-02-18T06:48:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2347420672 bytes 2026-02-18T06:48:35.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2384286720 bytes 2026-02-18T06:48:35.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2410261504 bytes 2026-02-18T06:48:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2437673984 bytes 2026-02-18T06:48:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2503972864 bytes 2026-02-18T06:48:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2588736512 bytes 2026-02-18T06:48:35.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2589352960 bytes 2026-02-18T06:48:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2681221120 bytes 2026-02-18T06:48:35.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2692737024 bytes 2026-02-18T06:48:35.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2764483584 bytes 2026-02-18T06:48:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2809542656 bytes 2026-02-18T06:48:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2834812928 bytes 2026-02-18T06:48:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2906535936 bytes 2026-02-18T06:48:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2996708352 bytes 2026-02-18T06:48:36.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3020529664 bytes 2026-02-18T06:48:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3085469696 bytes 2026-02-18T06:48:36.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3159472128 bytes 2026-02-18T06:48:36.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3257347072 bytes 2026-02-18T06:48:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3291142144 bytes 2026-02-18T06:48:36.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3346923520 bytes 2026-02-18T06:48:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3394963456 bytes 2026-02-18T06:48:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3395871744 bytes 2026-02-18T06:48:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3410326528 bytes 2026-02-18T06:48:36.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3440550912 bytes 2026-02-18T06:48:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3480298496 bytes 2026-02-18T06:48:37.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3556179968 bytes 2026-02-18T06:48:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3586708480 bytes 2026-02-18T06:48:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3640432640 bytes 2026-02-18T06:48:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3670923264 bytes 2026-02-18T06:48:37.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3671548928 bytes 2026-02-18T06:48:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3749633024 bytes 2026-02-18T06:48:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3780737024 bytes 2026-02-18T06:48:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3837089792 bytes 2026-02-18T06:48:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3884389376 bytes 2026-02-18T06:48:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3905377280 bytes 2026-02-18T06:48:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3968392192 bytes 2026-02-18T06:48:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4006181888 bytes 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_17 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:38.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_17 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:48:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2026-02-18T06:48:38.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_17': 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 134e1948e86f 2026-02-18T06:48:39.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.134e1948e86f 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:38 2026 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:38 2026 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:38 2026 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b71799a9-93c2-42d8-b47e-5494b730ec39 2026-02-18T06:48:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-18T06:48:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:48:39.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T06:48:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 17922048 bytes 2026-02-18T06:48:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 30577664 bytes 2026-02-18T06:48:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 45936640 bytes 2026-02-18T06:48:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 136982528 bytes 2026-02-18T06:48:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 180751360 bytes 2026-02-18T06:48:39.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 181782528 bytes 2026-02-18T06:48:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 246166528 bytes 2026-02-18T06:48:39.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 292098048 bytes 2026-02-18T06:48:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 358847488 bytes 2026-02-18T06:48:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 359479296 bytes 2026-02-18T06:48:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 406432768 bytes 2026-02-18T06:48:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 431787008 bytes 2026-02-18T06:48:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 532439040 bytes 2026-02-18T06:48:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 562205696 bytes 2026-02-18T06:48:39.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 600317952 bytes 2026-02-18T06:48:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 664440832 bytes 2026-02-18T06:48:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 674547712 bytes 2026-02-18T06:48:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 713927680 bytes 2026-02-18T06:48:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 756380672 bytes 2026-02-18T06:48:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 788253696 bytes 2026-02-18T06:48:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 863148032 bytes 2026-02-18T06:48:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 912520192 bytes 2026-02-18T06:48:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 941779968 bytes 2026-02-18T06:48:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1020161024 bytes 2026-02-18T06:48:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1086433280 bytes 2026-02-18T06:48:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1140472832 bytes 2026-02-18T06:48:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1188424704 bytes 2026-02-18T06:48:40.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1215781888 bytes 2026-02-18T06:48:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1232262144 bytes 2026-02-18T06:48:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1241397248 bytes 2026-02-18T06:48:40.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1271169024 bytes 2026-02-18T06:48:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1319854080 bytes 2026-02-18T06:48:41.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1335650304 bytes 2026-02-18T06:48:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1413186560 bytes 2026-02-18T06:48:41.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1455336448 bytes 2026-02-18T06:48:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1541918720 bytes 2026-02-18T06:48:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1563868160 bytes 2026-02-18T06:48:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1611003904 bytes 2026-02-18T06:48:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1691951104 bytes 2026-02-18T06:48:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1705321472 bytes 2026-02-18T06:48:41.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1754045440 bytes 2026-02-18T06:48:41.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1818999808 bytes 2026-02-18T06:48:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1862053888 bytes 2026-02-18T06:48:41.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1887919104 bytes 2026-02-18T06:48:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1917171712 bytes 2026-02-18T06:48:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1953405952 bytes 2026-02-18T06:48:41.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1990013952 bytes 2026-02-18T06:48:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2045989888 bytes 2026-02-18T06:48:42.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2167950336 bytes 2026-02-18T06:48:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2176577536 bytes 2026-02-18T06:48:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2318833664 bytes 2026-02-18T06:48:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2353259520 bytes 2026-02-18T06:48:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2381654016 bytes 2026-02-18T06:48:42.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2429174784 bytes 2026-02-18T06:48:42.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2452292608 bytes 2026-02-18T06:48:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2501182464 bytes 2026-02-18T06:48:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2545374208 bytes 2026-02-18T06:48:42.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2591154176 bytes 2026-02-18T06:48:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2642014208 bytes 2026-02-18T06:48:42.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2673565696 bytes 2026-02-18T06:48:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2697372672 bytes 2026-02-18T06:48:42.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2741005312 bytes 2026-02-18T06:48:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2789475328 bytes 2026-02-18T06:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2805327872 bytes 2026-02-18T06:48:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2855414784 bytes 2026-02-18T06:48:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2867334144 bytes 2026-02-18T06:48:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2893828096 bytes 2026-02-18T06:48:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2950234112 bytes 2026-02-18T06:48:43.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2976313344 bytes 2026-02-18T06:48:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3049410560 bytes 2026-02-18T06:48:43.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3076644864 bytes 2026-02-18T06:48:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3122603008 bytes 2026-02-18T06:48:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3132168192 bytes 2026-02-18T06:48:43.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3156612096 bytes 2026-02-18T06:48:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3226350592 bytes 2026-02-18T06:48:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3330615296 bytes 2026-02-18T06:48:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3376119808 bytes 2026-02-18T06:48:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3397275648 bytes 2026-02-18T06:48:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3422009344 bytes 2026-02-18T06:48:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3469535232 bytes 2026-02-18T06:48:44.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3538563072 bytes 2026-02-18T06:48:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3594425344 bytes 2026-02-18T06:48:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3680519168 bytes 2026-02-18T06:48:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3706915840 bytes 2026-02-18T06:48:44.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3729656832 bytes 2026-02-18T06:48:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3801076736 bytes 2026-02-18T06:48:44.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3828615168 bytes 2026-02-18T06:48:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3859874816 bytes 2026-02-18T06:48:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3909381120 bytes 2026-02-18T06:48:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3941277696 bytes 2026-02-18T06:48:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4017085440 bytes 2026-02-18T06:48:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4083061760 bytes 2026-02-18T06:48:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4119508992 bytes 2026-02-18T06:48:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4178684928 bytes 2026-02-18T06:48:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4212463616 bytes 2026-02-18T06:48:45.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4234469376 bytes 2026-02-18T06:48:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4293374976 bytes 2026-02-18T06:48:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4336591872 bytes 2026-02-18T06:48:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4383144960 bytes 2026-02-18T06:48:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4387873792 bytes 2026-02-18T06:48:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4417429504 bytes 2026-02-18T06:48:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4452280320 bytes 2026-02-18T06:48:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4468000768 bytes 2026-02-18T06:48:45.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4495420416 bytes 2026-02-18T06:48:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4525255680 bytes 2026-02-18T06:48:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4558057472 bytes 2026-02-18T06:48:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4579412992 bytes 2026-02-18T06:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4638718976 bytes 2026-02-18T06:48:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4717218816 bytes 2026-02-18T06:48:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4788954112 bytes 2026-02-18T06:48:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4810820608 bytes 2026-02-18T06:48:45.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4870358016 bytes 2026-02-18T06:48:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4906551296 bytes 2026-02-18T06:48:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4926006272 bytes 2026-02-18T06:48:46.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4998629376 bytes 2026-02-18T06:48:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5061435392 bytes 2026-02-18T06:48:46.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5131297792 bytes 2026-02-18T06:48:46.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5158061056 bytes 2026-02-18T06:48:46.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5236746240 bytes 2026-02-18T06:48:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5284251648 bytes 2026-02-18T06:48:46.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5365763072 bytes 2026-02-18T06:48:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5417545728 bytes 2026-02-18T06:48:46.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5472731136 bytes 2026-02-18T06:48:46.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5518520320 bytes 2026-02-18T06:48:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5583110144 bytes 2026-02-18T06:48:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5652092928 bytes 2026-02-18T06:48:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5672916992 bytes 2026-02-18T06:48:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5738569728 bytes 2026-02-18T06:48:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5782045696 bytes 2026-02-18T06:48:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5792135168 bytes 2026-02-18T06:48:47.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5816108032 bytes 2026-02-18T06:48:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5872267264 bytes 2026-02-18T06:48:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5969682432 bytes 2026-02-18T06:48:47.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6004962304 bytes 2026-02-18T06:48:47.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6058627072 bytes 2026-02-18T06:48:47.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6092061696 bytes 2026-02-18T06:48:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6092655616 bytes 2026-02-18T06:48:47.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6179853312 bytes 2026-02-18T06:48:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6235593728 bytes 2026-02-18T06:48:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6288870400 bytes 2026-02-18T06:48:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6356430848 bytes 2026-02-18T06:48:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6386796544 bytes 2026-02-18T06:48:48.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6409389056 bytes 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_18 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2026-02-18T06:48:48.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_18 2026-02-18T06:48:48.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:48.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2026-02-18T06:48:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:49.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2026-02-18T06:48:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_18': 2026-02-18T06:48:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:49.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 135a51d0d14d 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.135a51d0d14d 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:48 2026 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:48 2026 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:48 2026 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 58feaf8b-9b70-499d-a121-13c9d6bfc92a 2026-02-18T06:48:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:48:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:48:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 607232 bytes 2026-02-18T06:48:49.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 78995456 bytes 2026-02-18T06:48:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 116130816 bytes 2026-02-18T06:48:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 140491776 bytes 2026-02-18T06:48:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 182703104 bytes 2026-02-18T06:48:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 234574848 bytes 2026-02-18T06:48:49.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 296333312 bytes 2026-02-18T06:48:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 342066176 bytes 2026-02-18T06:48:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 352396288 bytes 2026-02-18T06:48:49.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 382452736 bytes 2026-02-18T06:48:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 416224256 bytes 2026-02-18T06:48:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 458765312 bytes 2026-02-18T06:48:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 477836288 bytes 2026-02-18T06:48:49.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 483929088 bytes 2026-02-18T06:48:49.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.2731489059687905) > weight (0.25) 2026-02-18T06:48:49.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.26699064985422327) > weight (0.25) 2026-02-18T06:48:49.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8428518586404566) > weight (0.25) 2026-02-18T06:48:49.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4954011418872344) > weight (0.25) 2026-02-18T06:48:49.857 INFO:tasks.rbd_mirror_thrash:waiting for 95.07616815713513 secs before thrashing 2026-02-18T06:48:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 501619712 bytes 2026-02-18T06:48:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 524806144 bytes 2026-02-18T06:48:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 582363136 bytes 2026-02-18T06:48:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 647219200 bytes 2026-02-18T06:48:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 658335744 bytes 2026-02-18T06:48:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 681497600 bytes 2026-02-18T06:48:50.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 698284032 bytes 2026-02-18T06:48:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 699467776 bytes 2026-02-18T06:48:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 747593728 bytes 2026-02-18T06:48:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 748197888 bytes 2026-02-18T06:48:50.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 809398272 bytes 2026-02-18T06:48:50.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 863126528 bytes 2026-02-18T06:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 900766720 bytes 2026-02-18T06:48:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 919161856 bytes 2026-02-18T06:48:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 962729984 bytes 2026-02-18T06:48:50.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1046283264 bytes 2026-02-18T06:48:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1069682688 bytes 2026-02-18T06:48:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1100523520 bytes 2026-02-18T06:48:50.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1152153600 bytes 2026-02-18T06:48:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1159456768 bytes 2026-02-18T06:48:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1197761536 bytes 2026-02-18T06:48:51.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1244731392 bytes 2026-02-18T06:48:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1261182976 bytes 2026-02-18T06:48:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1296038912 bytes 2026-02-18T06:48:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1334999040 bytes 2026-02-18T06:48:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1392422912 bytes 2026-02-18T06:48:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1393033216 bytes 2026-02-18T06:48:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1404455936 bytes 2026-02-18T06:48:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1430679552 bytes 2026-02-18T06:48:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1441224704 bytes 2026-02-18T06:48:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1470718976 bytes 2026-02-18T06:48:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1526035456 bytes 2026-02-18T06:48:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1550612480 bytes 2026-02-18T06:48:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1572953088 bytes 2026-02-18T06:48:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1608188928 bytes 2026-02-18T06:48:51.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1672446976 bytes 2026-02-18T06:48:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1702140928 bytes 2026-02-18T06:48:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1742963712 bytes 2026-02-18T06:48:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1813334016 bytes 2026-02-18T06:48:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1849532416 bytes 2026-02-18T06:48:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 1892135936 bytes 2026-02-18T06:48:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 1916539904 bytes 2026-02-18T06:48:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 1934138368 bytes 2026-02-18T06:48:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2005121024 bytes 2026-02-18T06:48:52.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2073154560 bytes 2026-02-18T06:48:52.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2139939840 bytes 2026-02-18T06:48:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2172436480 bytes 2026-02-18T06:48:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2192963584 bytes 2026-02-18T06:48:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2291705856 bytes 2026-02-18T06:48:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2319793152 bytes 2026-02-18T06:48:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2357304320 bytes 2026-02-18T06:48:52.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2368256000 bytes 2026-02-18T06:48:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2408219648 bytes 2026-02-18T06:48:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2459534336 bytes 2026-02-18T06:48:52.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2501049344 bytes 2026-02-18T06:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2528130048 bytes 2026-02-18T06:48:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2566632448 bytes 2026-02-18T06:48:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2624891904 bytes 2026-02-18T06:48:53.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2625530880 bytes 2026-02-18T06:48:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2651740160 bytes 2026-02-18T06:48:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2668847104 bytes 2026-02-18T06:48:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2696714240 bytes 2026-02-18T06:48:53.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2734602240 bytes 2026-02-18T06:48:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2751016960 bytes 2026-02-18T06:48:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2852647936 bytes 2026-02-18T06:48:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2899140608 bytes 2026-02-18T06:48:53.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 2899756032 bytes 2026-02-18T06:48:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 2934405120 bytes 2026-02-18T06:48:53.702 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:48:53.701+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbcf7425b0 2026-02-18T06:48:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 2969773056 bytes 2026-02-18T06:48:53.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 2976969728 bytes 2026-02-18T06:48:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 2992789504 bytes 2026-02-18T06:48:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3005546496 bytes 2026-02-18T06:48:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3022608384 bytes 2026-02-18T06:48:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3029424128 bytes 2026-02-18T06:48:53.930 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:48:53.928+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbcf7425b0 2026-02-18T06:48:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3063733248 bytes 2026-02-18T06:48:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3133248512 bytes 2026-02-18T06:48:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3143671808 bytes 2026-02-18T06:48:54.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3168509952 bytes 2026-02-18T06:48:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3222078464 bytes 2026-02-18T06:48:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3242568704 bytes 2026-02-18T06:48:54.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3265760256 bytes 2026-02-18T06:48:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3319817216 bytes 2026-02-18T06:48:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3363208192 bytes 2026-02-18T06:48:54.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3404351488 bytes 2026-02-18T06:48:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3522762752 bytes 2026-02-18T06:48:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3595000832 bytes 2026-02-18T06:48:54.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3642984448 bytes 2026-02-18T06:48:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3659217920 bytes 2026-02-18T06:48:54.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3710643200 bytes 2026-02-18T06:48:54.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 3788781568 bytes 2026-02-18T06:48:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 3850707968 bytes 2026-02-18T06:48:55.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 3897555968 bytes 2026-02-18T06:48:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 3959666688 bytes 2026-02-18T06:48:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4010556416 bytes 2026-02-18T06:48:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4027387904 bytes 2026-02-18T06:48:55.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4108645376 bytes 2026-02-18T06:48:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4134200320 bytes 2026-02-18T06:48:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4195214336 bytes 2026-02-18T06:48:55.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4201321472 bytes 2026-02-18T06:48:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4277720064 bytes 2026-02-18T06:48:55.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4316456960 bytes 2026-02-18T06:48:55.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4326645760 bytes 2026-02-18T06:48:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4345809920 bytes 2026-02-18T06:48:55.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4397199360 bytes 2026-02-18T06:48:55.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4408408064 bytes 2026-02-18T06:48:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4438394880 bytes 2026-02-18T06:48:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4481868800 bytes 2026-02-18T06:48:55.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4533229568 bytes 2026-02-18T06:48:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4557012992 bytes 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_19 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2026-02-18T06:48:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:48:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_19 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:48:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2026-02-18T06:48:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:48:57.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_19': 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1366477e771a 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1366477e771a 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:56 2026 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:56 2026 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:56 2026 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b90f47da-e462-47c3-9ed6-43d166775473 2026-02-18T06:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:48:57.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:48:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:48:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:48:57.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:48:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 49565696 bytes 2026-02-18T06:48:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 53544960 bytes 2026-02-18T06:48:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 112021504 bytes 2026-02-18T06:48:57.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 126763008 bytes 2026-02-18T06:48:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 165331968 bytes 2026-02-18T06:48:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 169570304 bytes 2026-02-18T06:48:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 250489856 bytes 2026-02-18T06:48:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 288372736 bytes 2026-02-18T06:48:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 296787968 bytes 2026-02-18T06:48:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 347505664 bytes 2026-02-18T06:48:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 350505984 bytes 2026-02-18T06:48:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 405486592 bytes 2026-02-18T06:48:57.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 425900032 bytes 2026-02-18T06:48:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 493490176 bytes 2026-02-18T06:48:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 562488320 bytes 2026-02-18T06:48:58.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 580446208 bytes 2026-02-18T06:48:58.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 654738432 bytes 2026-02-18T06:48:58.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 727051264 bytes 2026-02-18T06:48:58.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 751095808 bytes 2026-02-18T06:48:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 791881728 bytes 2026-02-18T06:48:58.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 826175488 bytes 2026-02-18T06:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 877617152 bytes 2026-02-18T06:48:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 890786816 bytes 2026-02-18T06:48:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 894084096 bytes 2026-02-18T06:48:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 936554496 bytes 2026-02-18T06:48:58.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 995068928 bytes 2026-02-18T06:48:58.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1079029760 bytes 2026-02-18T06:48:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1095004160 bytes 2026-02-18T06:48:58.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1109223424 bytes 2026-02-18T06:48:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1119669248 bytes 2026-02-18T06:48:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1133425664 bytes 2026-02-18T06:48:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1156536320 bytes 2026-02-18T06:48:59.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1164696576 bytes 2026-02-18T06:48:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1196691456 bytes 2026-02-18T06:48:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1269691392 bytes 2026-02-18T06:48:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1288873984 bytes 2026-02-18T06:48:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1340076032 bytes 2026-02-18T06:48:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1404103680 bytes 2026-02-18T06:48:59.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1419933696 bytes 2026-02-18T06:48:59.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1454768128 bytes 2026-02-18T06:48:59.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1523899392 bytes 2026-02-18T06:48:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1562086400 bytes 2026-02-18T06:48:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1613406208 bytes 2026-02-18T06:48:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1613996032 bytes 2026-02-18T06:48:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1642519552 bytes 2026-02-18T06:48:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1657975808 bytes 2026-02-18T06:48:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1683738624 bytes 2026-02-18T06:48:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1742019584 bytes 2026-02-18T06:48:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1773505536 bytes 2026-02-18T06:49:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1795658752 bytes 2026-02-18T06:49:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1842767872 bytes 2026-02-18T06:49:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1870262272 bytes 2026-02-18T06:49:00.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1907388416 bytes 2026-02-18T06:49:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1946796032 bytes 2026-02-18T06:49:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 1986926592 bytes 2026-02-18T06:49:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 1987506176 bytes 2026-02-18T06:49:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2012009472 bytes 2026-02-18T06:49:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2050861056 bytes 2026-02-18T06:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2114462720 bytes 2026-02-18T06:49:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2150085632 bytes 2026-02-18T06:49:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2178288640 bytes 2026-02-18T06:49:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2230061056 bytes 2026-02-18T06:49:00.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2266685440 bytes 2026-02-18T06:49:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2327286784 bytes 2026-02-18T06:49:01.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2445173760 bytes 2026-02-18T06:49:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2466533376 bytes 2026-02-18T06:49:01.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2572064768 bytes 2026-02-18T06:49:01.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2593032192 bytes 2026-02-18T06:49:01.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2614424576 bytes 2026-02-18T06:49:01.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2637418496 bytes 2026-02-18T06:49:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2670816256 bytes 2026-02-18T06:49:01.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2750126080 bytes 2026-02-18T06:49:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2750744576 bytes 2026-02-18T06:49:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2765254656 bytes 2026-02-18T06:49:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2788065280 bytes 2026-02-18T06:49:01.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2803818496 bytes 2026-02-18T06:49:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2844883968 bytes 2026-02-18T06:49:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2909140992 bytes 2026-02-18T06:49:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2953169920 bytes 2026-02-18T06:49:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2991791104 bytes 2026-02-18T06:49:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3005170688 bytes 2026-02-18T06:49:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3017569280 bytes 2026-02-18T06:49:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3053207552 bytes 2026-02-18T06:49:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3138209792 bytes 2026-02-18T06:49:02.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3173322752 bytes 2026-02-18T06:49:02.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3273805824 bytes 2026-02-18T06:49:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3301225472 bytes 2026-02-18T06:49:02.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3375393792 bytes 2026-02-18T06:49:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3441937408 bytes 2026-02-18T06:49:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3461042176 bytes 2026-02-18T06:49:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3510600704 bytes 2026-02-18T06:49:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3550260224 bytes 2026-02-18T06:49:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3607335936 bytes 2026-02-18T06:49:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3694551040 bytes 2026-02-18T06:49:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3736655872 bytes 2026-02-18T06:49:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3743408128 bytes 2026-02-18T06:49:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3776933888 bytes 2026-02-18T06:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:49:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_20 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:49:03.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:49:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_20 2026-02-18T06:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2026-02-18T06:49:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:49:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:03.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2026-02-18T06:49:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:49:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2026-02-18T06:49:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_20': 2026-02-18T06:49:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:49:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1372b441f92d 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1372b441f92d 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:03 2026 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:03 2026 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:03 2026 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:49:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:49:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 484fe09b-1bcd-49b2-9d6f-6019db73c59b 2026-02-18T06:49:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:49:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:49:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-18T06:49:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:49:05.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:49:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:49:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:49:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:49:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 52171776 bytes 2026-02-18T06:49:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 81835008 bytes 2026-02-18T06:49:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 82460672 bytes 2026-02-18T06:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 128592896 bytes 2026-02-18T06:49:05.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 130462720 bytes 2026-02-18T06:49:05.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 154432512 bytes 2026-02-18T06:49:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 187816960 bytes 2026-02-18T06:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 253186048 bytes 2026-02-18T06:49:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 294225920 bytes 2026-02-18T06:49:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 305729536 bytes 2026-02-18T06:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 323690496 bytes 2026-02-18T06:49:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 341083136 bytes 2026-02-18T06:49:05.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 350022656 bytes 2026-02-18T06:49:05.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 371311616 bytes 2026-02-18T06:49:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 376170496 bytes 2026-02-18T06:49:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 409725952 bytes 2026-02-18T06:49:06.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 438890496 bytes 2026-02-18T06:49:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 472240128 bytes 2026-02-18T06:49:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 546609152 bytes 2026-02-18T06:49:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 598950912 bytes 2026-02-18T06:49:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 603305984 bytes 2026-02-18T06:49:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 627188736 bytes 2026-02-18T06:49:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 683128832 bytes 2026-02-18T06:49:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 704581632 bytes 2026-02-18T06:49:06.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 719826944 bytes 2026-02-18T06:49:06.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 763193344 bytes 2026-02-18T06:49:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 825012224 bytes 2026-02-18T06:49:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 868124672 bytes 2026-02-18T06:49:06.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 932751360 bytes 2026-02-18T06:49:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 978164736 bytes 2026-02-18T06:49:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1039162368 bytes 2026-02-18T06:49:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1137645568 bytes 2026-02-18T06:49:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1216618496 bytes 2026-02-18T06:49:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1227024384 bytes 2026-02-18T06:49:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1257438208 bytes 2026-02-18T06:49:07.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1287444480 bytes 2026-02-18T06:49:07.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1328890880 bytes 2026-02-18T06:49:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1353951232 bytes 2026-02-18T06:49:07.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1379853312 bytes 2026-02-18T06:49:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1398452224 bytes 2026-02-18T06:49:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1454604288 bytes 2026-02-18T06:49:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1495868416 bytes 2026-02-18T06:49:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1511202816 bytes 2026-02-18T06:49:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1550224384 bytes 2026-02-18T06:49:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1571266560 bytes 2026-02-18T06:49:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1590418432 bytes 2026-02-18T06:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1646068736 bytes 2026-02-18T06:49:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1664580608 bytes 2026-02-18T06:49:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1715302400 bytes 2026-02-18T06:49:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1764191232 bytes 2026-02-18T06:49:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1802768384 bytes 2026-02-18T06:49:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1893832704 bytes 2026-02-18T06:49:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1962043392 bytes 2026-02-18T06:49:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2027154432 bytes 2026-02-18T06:49:08.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2072014848 bytes 2026-02-18T06:49:08.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2095199232 bytes 2026-02-18T06:49:08.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2185073664 bytes 2026-02-18T06:49:08.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2223338496 bytes 2026-02-18T06:49:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2242782208 bytes 2026-02-18T06:49:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2288249856 bytes 2026-02-18T06:49:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2310142976 bytes 2026-02-18T06:49:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2321588224 bytes 2026-02-18T06:49:08.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2342616064 bytes 2026-02-18T06:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2366243840 bytes 2026-02-18T06:49:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2366855168 bytes 2026-02-18T06:49:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2388884480 bytes 2026-02-18T06:49:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2420581376 bytes 2026-02-18T06:49:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2446938112 bytes 2026-02-18T06:49:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2447549440 bytes 2026-02-18T06:49:09.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2527157248 bytes 2026-02-18T06:49:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2607026176 bytes 2026-02-18T06:49:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2607646720 bytes 2026-02-18T06:49:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2609239040 bytes 2026-02-18T06:49:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2683204608 bytes 2026-02-18T06:49:09.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2683820032 bytes 2026-02-18T06:49:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2741468160 bytes 2026-02-18T06:49:09.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2789866496 bytes 2026-02-18T06:49:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2850961408 bytes 2026-02-18T06:49:09.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2856323072 bytes 2026-02-18T06:49:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2943222784 bytes 2026-02-18T06:49:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 2990116864 bytes 2026-02-18T06:49:09.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3014084608 bytes 2026-02-18T06:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3066362880 bytes 2026-02-18T06:49:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3094940672 bytes 2026-02-18T06:49:10.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3175965696 bytes 2026-02-18T06:49:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3216396288 bytes 2026-02-18T06:49:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3246596096 bytes 2026-02-18T06:49:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3314925568 bytes 2026-02-18T06:49:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3353189376 bytes 2026-02-18T06:49:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3419790336 bytes 2026-02-18T06:49:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3453095936 bytes 2026-02-18T06:49:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3504212992 bytes 2026-02-18T06:49:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3561573376 bytes 2026-02-18T06:49:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3585157120 bytes 2026-02-18T06:49:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3606690816 bytes 2026-02-18T06:49:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3686373376 bytes 2026-02-18T06:49:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3696103424 bytes 2026-02-18T06:49:10.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3705211904 bytes 2026-02-18T06:49:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3768911872 bytes 2026-02-18T06:49:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3785779200 bytes 2026-02-18T06:49:11.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3824742400 bytes 2026-02-18T06:49:11.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3906743296 bytes 2026-02-18T06:49:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3985974272 bytes 2026-02-18T06:49:11.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4005737472 bytes 2026-02-18T06:49:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4069557248 bytes 2026-02-18T06:49:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4098390016 bytes 2026-02-18T06:49:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4123447296 bytes 2026-02-18T06:49:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4142693376 bytes 2026-02-18T06:49:11.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4162572288 bytes 2026-02-18T06:49:11.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4197600256 bytes 2026-02-18T06:49:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4300523520 bytes 2026-02-18T06:49:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4311508992 bytes 2026-02-18T06:49:11.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4343289856 bytes 2026-02-18T06:49:11.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4372541440 bytes 2026-02-18T06:49:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4439727104 bytes 2026-02-18T06:49:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4510882816 bytes 2026-02-18T06:49:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4563092480 bytes 2026-02-18T06:49:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4605255680 bytes 2026-02-18T06:49:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4606568448 bytes 2026-02-18T06:49:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4664216576 bytes 2026-02-18T06:49:12.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4699517952 bytes 2026-02-18T06:49:12.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4719506432 bytes 2026-02-18T06:49:12.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4739325952 bytes 2026-02-18T06:49:12.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4774260736 bytes 2026-02-18T06:49:12.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4783259648 bytes 2026-02-18T06:49:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 4839285760 bytes 2026-02-18T06:49:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 4888581120 bytes 2026-02-18T06:49:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 4914033664 bytes 2026-02-18T06:49:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 4947985408 bytes 2026-02-18T06:49:12.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 4956026880 bytes 2026-02-18T06:49:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5008752640 bytes 2026-02-18T06:49:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5027759104 bytes 2026-02-18T06:49:12.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5069868032 bytes 2026-02-18T06:49:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5130513408 bytes 2026-02-18T06:49:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5159487488 bytes 2026-02-18T06:49:13.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5191019520 bytes 2026-02-18T06:49:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5191625728 bytes 2026-02-18T06:49:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5300398080 bytes 2026-02-18T06:49:13.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5321450496 bytes 2026-02-18T06:49:13.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5361904640 bytes 2026-02-18T06:49:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5413876736 bytes 2026-02-18T06:49:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5454701568 bytes 2026-02-18T06:49:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5477282816 bytes 2026-02-18T06:49:13.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5534849024 bytes 2026-02-18T06:49:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5574238208 bytes 2026-02-18T06:49:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5633012736 bytes 2026-02-18T06:49:13.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5678453760 bytes 2026-02-18T06:49:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5710512128 bytes 2026-02-18T06:49:13.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5775784960 bytes 2026-02-18T06:49:14.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 5777682432 bytes 2026-02-18T06:49:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 5825527808 bytes 2026-02-18T06:49:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 5828648960 bytes 2026-02-18T06:49:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 5897568256 bytes 2026-02-18T06:49:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 5921222656 bytes 2026-02-18T06:49:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 5963953152 bytes 2026-02-18T06:49:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6008782848 bytes 2026-02-18T06:49:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6036926464 bytes 2026-02-18T06:49:14.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6056704000 bytes 2026-02-18T06:49:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6126120960 bytes 2026-02-18T06:49:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6149937152 bytes 2026-02-18T06:49:14.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6226475008 bytes 2026-02-18T06:49:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6289028096 bytes 2026-02-18T06:49:14.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6325005312 bytes 2026-02-18T06:49:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6405981184 bytes 2026-02-18T06:49:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6446697472 bytes 2026-02-18T06:49:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6532324352 bytes 2026-02-18T06:49:15.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.26536218457157246) > weight (0.25) 2026-02-18T06:49:15.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8632233793009408) > weight (0.25) 2026-02-18T06:49:15.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6595036673706076) > weight (0.25) 2026-02-18T06:49:15.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.43041252593451385) > weight (0.25) 2026-02-18T06:49:15.054 INFO:tasks.rbd_mirror_thrash:waiting for 85.58939713215801 secs before thrashing 2026-02-18T06:49:15.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6552181760 bytes 2026-02-18T06:49:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6597158912 bytes 2026-02-18T06:49:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6643062784 bytes 2026-02-18T06:49:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6709842944 bytes 2026-02-18T06:49:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6756049920 bytes 2026-02-18T06:49:15.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6796119040 bytes 2026-02-18T06:49:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6813839360 bytes 2026-02-18T06:49:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6912607232 bytes 2026-02-18T06:49:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 6950682624 bytes 2026-02-18T06:49:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7093924864 bytes 2026-02-18T06:49:15.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7155973120 bytes 2026-02-18T06:49:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7156576256 bytes 2026-02-18T06:49:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7200150528 bytes 2026-02-18T06:49:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7204936704 bytes 2026-02-18T06:49:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7239298048 bytes 2026-02-18T06:49:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7308271616 bytes 2026-02-18T06:49:16.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7357932544 bytes 2026-02-18T06:49:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7406734336 bytes 2026-02-18T06:49:16.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7432179712 bytes 2026-02-18T06:49:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7459216384 bytes 2026-02-18T06:49:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7515943936 bytes 2026-02-18T06:49:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7522027520 bytes 2026-02-18T06:49:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7627708416 bytes 2026-02-18T06:49:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7726462976 bytes 2026-02-18T06:49:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7734934528 bytes 2026-02-18T06:49:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7795688448 bytes 2026-02-18T06:49:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7821140992 bytes 2026-02-18T06:49:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7870077952 bytes 2026-02-18T06:49:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7876871168 bytes 2026-02-18T06:49:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7904039936 bytes 2026-02-18T06:49:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7993995264 bytes 2026-02-18T06:49:17.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8009306112 bytes 2026-02-18T06:49:17.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8019214336 bytes 2026-02-18T06:49:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8019853312 bytes 2026-02-18T06:49:17.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8114506752 bytes 2026-02-18T06:49:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8134517760 bytes 2026-02-18T06:49:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8141989888 bytes 2026-02-18T06:49:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8174913536 bytes 2026-02-18T06:49:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8198864896 bytes 2026-02-18T06:49:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8235774976 bytes 2026-02-18T06:49:17.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8250450944 bytes 2026-02-18T06:49:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8274868224 bytes 2026-02-18T06:49:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8322583552 bytes 2026-02-18T06:49:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8355630080 bytes 2026-02-18T06:49:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8380298240 bytes 2026-02-18T06:49:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8458608640 bytes 2026-02-18T06:49:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 8474305536 bytes 2026-02-18T06:49:17.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 8519716864 bytes 2026-02-18T06:49:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 8539924480 bytes 2026-02-18T06:49:18.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 8607702016 bytes 2026-02-18T06:49:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21700 IOs, 8653922304 bytes 2026-02-18T06:49:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21800 IOs, 8689552384 bytes 2026-02-18T06:49:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21900 IOs, 8732491776 bytes 2026-02-18T06:49:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22000 IOs, 8809158656 bytes 2026-02-18T06:49:18.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22100 IOs, 8896904192 bytes 2026-02-18T06:49:18.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22200 IOs, 8967905280 bytes 2026-02-18T06:49:18.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22300 IOs, 9003619328 bytes 2026-02-18T06:49:18.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22400 IOs, 9082509312 bytes 2026-02-18T06:49:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22500 IOs, 9110351872 bytes 2026-02-18T06:49:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22600 IOs, 9126881280 bytes 2026-02-18T06:49:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22700 IOs, 9149127680 bytes 2026-02-18T06:49:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22800 IOs, 9149755392 bytes 2026-02-18T06:49:18.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:22900 IOs, 9250937856 bytes 2026-02-18T06:49:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:23000 IOs, 9317486592 bytes 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:49:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_21 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_21 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:49:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2026-02-18T06:49:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:49:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_21': 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 137e5ea89594 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.137e5ea89594 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:19 2026 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:19 2026 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:19 2026 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3ecdab12-dc8c-4836-8efc-36c5c66c919a 2026-02-18T06:49:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:49:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:49:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:49:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:49:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:49:20.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 57604096 bytes 2026-02-18T06:49:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 95022080 bytes 2026-02-18T06:49:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 189153280 bytes 2026-02-18T06:49:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 270300160 bytes 2026-02-18T06:49:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 334319616 bytes 2026-02-18T06:49:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 334971904 bytes 2026-02-18T06:49:20.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 377126912 bytes 2026-02-18T06:49:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 478331904 bytes 2026-02-18T06:49:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 482299904 bytes 2026-02-18T06:49:20.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 520455168 bytes 2026-02-18T06:49:20.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 586839040 bytes 2026-02-18T06:49:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 593605632 bytes 2026-02-18T06:49:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 640811008 bytes 2026-02-18T06:49:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 655221760 bytes 2026-02-18T06:49:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 712441856 bytes 2026-02-18T06:49:21.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 786064384 bytes 2026-02-18T06:49:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 859888640 bytes 2026-02-18T06:49:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 882928640 bytes 2026-02-18T06:49:21.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 925318144 bytes 2026-02-18T06:49:21.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 977965056 bytes 2026-02-18T06:49:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1033774080 bytes 2026-02-18T06:49:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1089828864 bytes 2026-02-18T06:49:21.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1153357824 bytes 2026-02-18T06:49:21.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1256515584 bytes 2026-02-18T06:49:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1296728064 bytes 2026-02-18T06:49:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1318276096 bytes 2026-02-18T06:49:22.018 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.016+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f5a07c70 2026-02-18T06:49:22.019 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.016+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f7096000 2026-02-18T06:49:22.020 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.018+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f7096820 2026-02-18T06:49:22.020 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.018+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f9092b60 2026-02-18T06:49:22.020 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.018+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f9dca4e0 2026-02-18T06:49:22.020 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:49:22.018+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fa363520 2026-02-18T06:49:22.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1358398464 bytes 2026-02-18T06:49:22.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1438828544 bytes 2026-02-18T06:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1481659392 bytes 2026-02-18T06:49:22.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1512641536 bytes 2026-02-18T06:49:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1541711872 bytes 2026-02-18T06:49:22.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1557562368 bytes 2026-02-18T06:49:22.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1607285760 bytes 2026-02-18T06:49:22.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1636557824 bytes 2026-02-18T06:49:22.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1663961088 bytes 2026-02-18T06:49:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1681498112 bytes 2026-02-18T06:49:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1702158336 bytes 2026-02-18T06:49:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1777954816 bytes 2026-02-18T06:49:22.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1808351232 bytes 2026-02-18T06:49:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1816290304 bytes 2026-02-18T06:49:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1849608192 bytes 2026-02-18T06:49:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1897923584 bytes 2026-02-18T06:49:23.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1983223808 bytes 2026-02-18T06:49:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2048594944 bytes 2026-02-18T06:49:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2075542528 bytes 2026-02-18T06:49:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2081045504 bytes 2026-02-18T06:49:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2081637376 bytes 2026-02-18T06:49:23.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2085757952 bytes 2026-02-18T06:49:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2114720768 bytes 2026-02-18T06:49:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2136179712 bytes 2026-02-18T06:49:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2136767488 bytes 2026-02-18T06:49:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2167393280 bytes 2026-02-18T06:49:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2209680384 bytes 2026-02-18T06:49:23.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2288522240 bytes 2026-02-18T06:49:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2300255232 bytes 2026-02-18T06:49:23.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2365401088 bytes 2026-02-18T06:49:23.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2387779584 bytes 2026-02-18T06:49:23.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2425420800 bytes 2026-02-18T06:49:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2500150272 bytes 2026-02-18T06:49:23.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2551763968 bytes 2026-02-18T06:49:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2660116480 bytes 2026-02-18T06:49:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2687365120 bytes 2026-02-18T06:49:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2770816000 bytes 2026-02-18T06:49:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2825351168 bytes 2026-02-18T06:49:24.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2876496896 bytes 2026-02-18T06:49:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2899924992 bytes 2026-02-18T06:49:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2924659712 bytes 2026-02-18T06:49:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2984993792 bytes 2026-02-18T06:49:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3010972672 bytes 2026-02-18T06:49:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3047549952 bytes 2026-02-18T06:49:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3079394304 bytes 2026-02-18T06:49:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3132640256 bytes 2026-02-18T06:49:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3193677824 bytes 2026-02-18T06:49:24.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3219103744 bytes 2026-02-18T06:49:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3266079744 bytes 2026-02-18T06:49:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3307582464 bytes 2026-02-18T06:49:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3353677824 bytes 2026-02-18T06:49:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3400032256 bytes 2026-02-18T06:49:25.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3439377408 bytes 2026-02-18T06:49:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3547397120 bytes 2026-02-18T06:49:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3650942976 bytes 2026-02-18T06:49:25.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3656014848 bytes 2026-02-18T06:49:25.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3713864704 bytes 2026-02-18T06:49:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3729290240 bytes 2026-02-18T06:49:25.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3789265920 bytes 2026-02-18T06:49:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3840309248 bytes 2026-02-18T06:49:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3878853632 bytes 2026-02-18T06:49:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3958979584 bytes 2026-02-18T06:49:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4013163520 bytes 2026-02-18T06:49:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4048739328 bytes 2026-02-18T06:49:26.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4108968960 bytes 2026-02-18T06:49:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4155253760 bytes 2026-02-18T06:49:26.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4188951552 bytes 2026-02-18T06:49:26.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4241275904 bytes 2026-02-18T06:49:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4291366912 bytes 2026-02-18T06:49:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4344524800 bytes 2026-02-18T06:49:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4393299968 bytes 2026-02-18T06:49:26.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4479797248 bytes 2026-02-18T06:49:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4495835136 bytes 2026-02-18T06:49:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4562868224 bytes 2026-02-18T06:49:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4596610048 bytes 2026-02-18T06:49:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4642719744 bytes 2026-02-18T06:49:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4701418496 bytes 2026-02-18T06:49:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4750431232 bytes 2026-02-18T06:49:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4779709440 bytes 2026-02-18T06:49:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4812926976 bytes 2026-02-18T06:49:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4877170688 bytes 2026-02-18T06:49:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4919844864 bytes 2026-02-18T06:49:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4963685376 bytes 2026-02-18T06:49:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5028058112 bytes 2026-02-18T06:49:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5080431616 bytes 2026-02-18T06:49:27.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5124572160 bytes 2026-02-18T06:49:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5179470848 bytes 2026-02-18T06:49:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5180293120 bytes 2026-02-18T06:49:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5296274432 bytes 2026-02-18T06:49:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5302390784 bytes 2026-02-18T06:49:27.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5348146176 bytes 2026-02-18T06:49:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5399545856 bytes 2026-02-18T06:49:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5517572096 bytes 2026-02-18T06:49:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5538314240 bytes 2026-02-18T06:49:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5573892096 bytes 2026-02-18T06:49:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5613022208 bytes 2026-02-18T06:49:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5639571456 bytes 2026-02-18T06:49:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5699318784 bytes 2026-02-18T06:49:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5733949440 bytes 2026-02-18T06:49:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5801461760 bytes 2026-02-18T06:49:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5833342976 bytes 2026-02-18T06:49:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5856320512 bytes 2026-02-18T06:49:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5856928768 bytes 2026-02-18T06:49:28.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5894941696 bytes 2026-02-18T06:49:28.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5934576640 bytes 2026-02-18T06:49:28.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5964259328 bytes 2026-02-18T06:49:28.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6000132096 bytes 2026-02-18T06:49:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6045419520 bytes 2026-02-18T06:49:28.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6095750144 bytes 2026-02-18T06:49:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6141668352 bytes 2026-02-18T06:49:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6185797632 bytes 2026-02-18T06:49:28.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6231240704 bytes 2026-02-18T06:49:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6246820864 bytes 2026-02-18T06:49:29.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6293993472 bytes 2026-02-18T06:49:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6347572224 bytes 2026-02-18T06:49:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6349147136 bytes 2026-02-18T06:49:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6369768448 bytes 2026-02-18T06:49:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6423128064 bytes 2026-02-18T06:49:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6471739392 bytes 2026-02-18T06:49:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6509959168 bytes 2026-02-18T06:49:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6523046912 bytes 2026-02-18T06:49:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6595949568 bytes 2026-02-18T06:49:29.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6670493696 bytes 2026-02-18T06:49:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6688163840 bytes 2026-02-18T06:49:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6729940992 bytes 2026-02-18T06:49:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6804196352 bytes 2026-02-18T06:49:29.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6824154112 bytes 2026-02-18T06:49:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6844055552 bytes 2026-02-18T06:49:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6906053632 bytes 2026-02-18T06:49:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6961593344 bytes 2026-02-18T06:49:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7002328064 bytes 2026-02-18T06:49:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7030108160 bytes 2026-02-18T06:49:30.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7096469504 bytes 2026-02-18T06:49:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7124535296 bytes 2026-02-18T06:49:30.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7166797824 bytes 2026-02-18T06:49:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7223281664 bytes 2026-02-18T06:49:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7285757952 bytes 2026-02-18T06:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7345638400 bytes 2026-02-18T06:49:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7386821632 bytes 2026-02-18T06:49:30.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7398465536 bytes 2026-02-18T06:49:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7409549312 bytes 2026-02-18T06:49:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7460471808 bytes 2026-02-18T06:49:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7520607232 bytes 2026-02-18T06:49:30.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7532993536 bytes 2026-02-18T06:49:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7586183168 bytes 2026-02-18T06:49:31.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7653187584 bytes 2026-02-18T06:49:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_22 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_22 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2026-02-18T06:49:31.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:49:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2026-02-18T06:49:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:49:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_22': 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 138ad85a8764 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.138ad85a8764 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:49:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:31 2026 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:31 2026 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:31 2026 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: caed0cd9-e185-4f04-9bf1-ef7abd391a34 2026-02-18T06:49:32.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:49:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:49:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-18T06:49:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:49:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:49:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:49:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:49:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:49:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 11896832 bytes 2026-02-18T06:49:32.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 76974080 bytes 2026-02-18T06:49:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 131315712 bytes 2026-02-18T06:49:32.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 169761792 bytes 2026-02-18T06:49:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 180965376 bytes 2026-02-18T06:49:32.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 224712704 bytes 2026-02-18T06:49:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 299482112 bytes 2026-02-18T06:49:32.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 301954048 bytes 2026-02-18T06:49:32.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 307814400 bytes 2026-02-18T06:49:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 321774592 bytes 2026-02-18T06:49:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 324921344 bytes 2026-02-18T06:49:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 402419712 bytes 2026-02-18T06:49:32.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 425650176 bytes 2026-02-18T06:49:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 472233984 bytes 2026-02-18T06:49:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 588393472 bytes 2026-02-18T06:49:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 650323968 bytes 2026-02-18T06:49:33.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 714308608 bytes 2026-02-18T06:49:33.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 763496448 bytes 2026-02-18T06:49:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 789451776 bytes 2026-02-18T06:49:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 844683264 bytes 2026-02-18T06:49:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 895441920 bytes 2026-02-18T06:49:33.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 919457792 bytes 2026-02-18T06:49:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 962744320 bytes 2026-02-18T06:49:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1033049088 bytes 2026-02-18T06:49:33.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1084272640 bytes 2026-02-18T06:49:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1112650752 bytes 2026-02-18T06:49:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1117669376 bytes 2026-02-18T06:49:33.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1158809600 bytes 2026-02-18T06:49:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1248174080 bytes 2026-02-18T06:49:33.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1261192192 bytes 2026-02-18T06:49:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1267876864 bytes 2026-02-18T06:49:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1310592000 bytes 2026-02-18T06:49:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1371463680 bytes 2026-02-18T06:49:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1404093440 bytes 2026-02-18T06:49:34.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1425090560 bytes 2026-02-18T06:49:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1452735488 bytes 2026-02-18T06:49:34.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1460235264 bytes 2026-02-18T06:49:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1460831232 bytes 2026-02-18T06:49:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1486249984 bytes 2026-02-18T06:49:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1506339840 bytes 2026-02-18T06:49:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1554488320 bytes 2026-02-18T06:49:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1670611968 bytes 2026-02-18T06:49:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1714976768 bytes 2026-02-18T06:49:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1737184256 bytes 2026-02-18T06:49:34.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1766299648 bytes 2026-02-18T06:49:34.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1801873408 bytes 2026-02-18T06:49:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1842583552 bytes 2026-02-18T06:49:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1915974656 bytes 2026-02-18T06:49:34.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1950932992 bytes 2026-02-18T06:49:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1974395904 bytes 2026-02-18T06:49:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2108380160 bytes 2026-02-18T06:49:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2109465600 bytes 2026-02-18T06:49:35.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2185064448 bytes 2026-02-18T06:49:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2303685632 bytes 2026-02-18T06:49:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2378648576 bytes 2026-02-18T06:49:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2437878784 bytes 2026-02-18T06:49:35.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2456571904 bytes 2026-02-18T06:49:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2496498688 bytes 2026-02-18T06:49:35.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2547247104 bytes 2026-02-18T06:49:35.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2569803776 bytes 2026-02-18T06:49:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2595214336 bytes 2026-02-18T06:49:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2654372864 bytes 2026-02-18T06:49:35.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2703819776 bytes 2026-02-18T06:49:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2756545536 bytes 2026-02-18T06:49:35.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2757177344 bytes 2026-02-18T06:49:36.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2817432576 bytes 2026-02-18T06:49:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2828178432 bytes 2026-02-18T06:49:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2892958720 bytes 2026-02-18T06:49:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2915402752 bytes 2026-02-18T06:49:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2948611072 bytes 2026-02-18T06:49:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2965689344 bytes 2026-02-18T06:49:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3006626816 bytes 2026-02-18T06:49:36.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3034728448 bytes 2026-02-18T06:49:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3068388352 bytes 2026-02-18T06:49:36.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3130390528 bytes 2026-02-18T06:49:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3195666432 bytes 2026-02-18T06:49:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3219473408 bytes 2026-02-18T06:49:36.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3220099072 bytes 2026-02-18T06:49:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3273539584 bytes 2026-02-18T06:49:36.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3330308096 bytes 2026-02-18T06:49:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3377188864 bytes 2026-02-18T06:49:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3422373888 bytes 2026-02-18T06:49:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3477554176 bytes 2026-02-18T06:49:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3566106624 bytes 2026-02-18T06:49:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3591861248 bytes 2026-02-18T06:49:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3636284416 bytes 2026-02-18T06:49:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3738744832 bytes 2026-02-18T06:49:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3774001152 bytes 2026-02-18T06:49:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3840155648 bytes 2026-02-18T06:49:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3907617792 bytes 2026-02-18T06:49:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3983300608 bytes 2026-02-18T06:49:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4047342592 bytes 2026-02-18T06:49:37.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4065893376 bytes 2026-02-18T06:49:37.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4097289216 bytes 2026-02-18T06:49:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4128223232 bytes 2026-02-18T06:49:37.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4177274880 bytes 2026-02-18T06:49:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4216245248 bytes 2026-02-18T06:49:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4244839424 bytes 2026-02-18T06:49:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:49:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_23 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:49:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:49:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:49:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_23 2026-02-18T06:49:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2026-02-18T06:49:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:49:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2026-02-18T06:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_23': 2026-02-18T06:49:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1396613ed03a 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1396613ed03a 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:38 2026 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:38 2026 2026-02-18T06:49:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:38 2026 2026-02-18T06:49:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:49:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:49:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3e439ed2-471c-4885-bdf3-4e6705e30c66 2026-02-18T06:49:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:49:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:49:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:49:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:49:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:49:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 11148288 bytes 2026-02-18T06:49:39.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 43536384 bytes 2026-02-18T06:49:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 86974464 bytes 2026-02-18T06:49:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 165756928 bytes 2026-02-18T06:49:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 214491136 bytes 2026-02-18T06:49:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 249359360 bytes 2026-02-18T06:49:39.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 294356992 bytes 2026-02-18T06:49:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 339075072 bytes 2026-02-18T06:49:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 376958976 bytes 2026-02-18T06:49:39.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 426358784 bytes 2026-02-18T06:49:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 470949888 bytes 2026-02-18T06:49:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 516741120 bytes 2026-02-18T06:49:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 566228992 bytes 2026-02-18T06:49:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 578331648 bytes 2026-02-18T06:49:39.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 632417280 bytes 2026-02-18T06:49:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 644080640 bytes 2026-02-18T06:49:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 697097216 bytes 2026-02-18T06:49:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 738214912 bytes 2026-02-18T06:49:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 777467904 bytes 2026-02-18T06:49:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 856014848 bytes 2026-02-18T06:49:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 902725632 bytes 2026-02-18T06:49:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 922500096 bytes 2026-02-18T06:49:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 947900416 bytes 2026-02-18T06:49:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 965891072 bytes 2026-02-18T06:49:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1004502016 bytes 2026-02-18T06:49:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1038151680 bytes 2026-02-18T06:49:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1085755392 bytes 2026-02-18T06:49:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1182571520 bytes 2026-02-18T06:49:40.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1199903744 bytes 2026-02-18T06:49:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1287318528 bytes 2026-02-18T06:49:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1356936192 bytes 2026-02-18T06:49:41.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1437807616 bytes 2026-02-18T06:49:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1475786752 bytes 2026-02-18T06:49:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1476382720 bytes 2026-02-18T06:49:41.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1536115712 bytes 2026-02-18T06:49:41.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1586787328 bytes 2026-02-18T06:49:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1614130176 bytes 2026-02-18T06:49:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1635489792 bytes 2026-02-18T06:49:41.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1785971712 bytes 2026-02-18T06:49:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1786552320 bytes 2026-02-18T06:49:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1842118656 bytes 2026-02-18T06:49:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1846681600 bytes 2026-02-18T06:49:41.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1935458304 bytes 2026-02-18T06:49:41.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1936092160 bytes 2026-02-18T06:49:41.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1987883008 bytes 2026-02-18T06:49:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2027252736 bytes 2026-02-18T06:49:42.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2100950016 bytes 2026-02-18T06:49:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2101551104 bytes 2026-02-18T06:49:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2155486208 bytes 2026-02-18T06:49:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2224518144 bytes 2026-02-18T06:49:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2290562048 bytes 2026-02-18T06:49:42.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2302666752 bytes 2026-02-18T06:49:42.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2314779648 bytes 2026-02-18T06:49:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2357962752 bytes 2026-02-18T06:49:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2413840384 bytes 2026-02-18T06:49:42.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2492233728 bytes 2026-02-18T06:49:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2533227520 bytes 2026-02-18T06:49:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2557435904 bytes 2026-02-18T06:49:42.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2622535680 bytes 2026-02-18T06:49:42.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2652173312 bytes 2026-02-18T06:49:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2677495808 bytes 2026-02-18T06:49:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2730215424 bytes 2026-02-18T06:49:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2809766912 bytes 2026-02-18T06:49:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2909022208 bytes 2026-02-18T06:49:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2942069760 bytes 2026-02-18T06:49:43.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2985219072 bytes 2026-02-18T06:49:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3024358400 bytes 2026-02-18T06:49:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3038783488 bytes 2026-02-18T06:49:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3070770176 bytes 2026-02-18T06:49:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3165313024 bytes 2026-02-18T06:49:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3211795456 bytes 2026-02-18T06:49:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3234746368 bytes 2026-02-18T06:49:43.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3242767360 bytes 2026-02-18T06:49:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3266023424 bytes 2026-02-18T06:49:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3275945984 bytes 2026-02-18T06:49:43.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3321831424 bytes 2026-02-18T06:49:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3349739520 bytes 2026-02-18T06:49:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3371702272 bytes 2026-02-18T06:49:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3405113344 bytes 2026-02-18T06:49:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3436977152 bytes 2026-02-18T06:49:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3457746944 bytes 2026-02-18T06:49:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3472101376 bytes 2026-02-18T06:49:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3540666368 bytes 2026-02-18T06:49:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3554981888 bytes 2026-02-18T06:49:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3630774272 bytes 2026-02-18T06:49:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3656288256 bytes 2026-02-18T06:49:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3656877056 bytes 2026-02-18T06:49:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3673829376 bytes 2026-02-18T06:49:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3780149248 bytes 2026-02-18T06:49:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3845175296 bytes 2026-02-18T06:49:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3882749952 bytes 2026-02-18T06:49:44.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3937474560 bytes 2026-02-18T06:49:44.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3958288384 bytes 2026-02-18T06:49:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3984975872 bytes 2026-02-18T06:49:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3999738880 bytes 2026-02-18T06:49:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4053370880 bytes 2026-02-18T06:49:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4099806208 bytes 2026-02-18T06:49:45.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4118508544 bytes 2026-02-18T06:49:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4137998336 bytes 2026-02-18T06:49:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4150335488 bytes 2026-02-18T06:49:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4159930368 bytes 2026-02-18T06:49:45.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4174113792 bytes 2026-02-18T06:49:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4209873920 bytes 2026-02-18T06:49:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4243304448 bytes 2026-02-18T06:49:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4272743424 bytes 2026-02-18T06:49:45.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4323930112 bytes 2026-02-18T06:49:45.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4385009664 bytes 2026-02-18T06:49:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4409305088 bytes 2026-02-18T06:49:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4450440192 bytes 2026-02-18T06:49:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4451039232 bytes 2026-02-18T06:49:45.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4529336320 bytes 2026-02-18T06:49:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4623376384 bytes 2026-02-18T06:49:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4663208960 bytes 2026-02-18T06:49:45.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4668326912 bytes 2026-02-18T06:49:45.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4714771456 bytes 2026-02-18T06:49:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4739912704 bytes 2026-02-18T06:49:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4814352384 bytes 2026-02-18T06:49:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4844935168 bytes 2026-02-18T06:49:46.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4889294848 bytes 2026-02-18T06:49:46.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4967584768 bytes 2026-02-18T06:49:46.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5031286784 bytes 2026-02-18T06:49:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5057241088 bytes 2026-02-18T06:49:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5135156224 bytes 2026-02-18T06:49:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5242605568 bytes 2026-02-18T06:49:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5268226048 bytes 2026-02-18T06:49:46.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5268842496 bytes 2026-02-18T06:49:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5298953216 bytes 2026-02-18T06:49:46.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5327212544 bytes 2026-02-18T06:49:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5385036800 bytes 2026-02-18T06:49:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5439714304 bytes 2026-02-18T06:49:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5489180672 bytes 2026-02-18T06:49:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5580601344 bytes 2026-02-18T06:49:47.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5613892608 bytes 2026-02-18T06:49:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5624283136 bytes 2026-02-18T06:49:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5640438784 bytes 2026-02-18T06:49:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5641079808 bytes 2026-02-18T06:49:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5705457664 bytes 2026-02-18T06:49:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5717081088 bytes 2026-02-18T06:49:47.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5731093504 bytes 2026-02-18T06:49:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5779254272 bytes 2026-02-18T06:49:47.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5831643136 bytes 2026-02-18T06:49:47.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5852227584 bytes 2026-02-18T06:49:47.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5866977280 bytes 2026-02-18T06:49:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5897472000 bytes 2026-02-18T06:49:47.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5944933376 bytes 2026-02-18T06:49:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5989357568 bytes 2026-02-18T06:49:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6044076032 bytes 2026-02-18T06:49:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6118862848 bytes 2026-02-18T06:49:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6142401536 bytes 2026-02-18T06:49:48.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6185264128 bytes 2026-02-18T06:49:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6248247296 bytes 2026-02-18T06:49:48.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6287074304 bytes 2026-02-18T06:49:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6346317824 bytes 2026-02-18T06:49:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6364087296 bytes 2026-02-18T06:49:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6397641728 bytes 2026-02-18T06:49:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6453344256 bytes 2026-02-18T06:49:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6466983936 bytes 2026-02-18T06:49:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6480031744 bytes 2026-02-18T06:49:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6498975744 bytes 2026-02-18T06:49:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6510177280 bytes 2026-02-18T06:49:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6592618496 bytes 2026-02-18T06:49:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6593232896 bytes 2026-02-18T06:49:48.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6641289216 bytes 2026-02-18T06:49:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6708563968 bytes 2026-02-18T06:49:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6745128960 bytes 2026-02-18T06:49:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6804648960 bytes 2026-02-18T06:49:49.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6831132672 bytes 2026-02-18T06:49:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6860954624 bytes 2026-02-18T06:49:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6943324160 bytes 2026-02-18T06:49:49.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6993583104 bytes 2026-02-18T06:49:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7054955520 bytes 2026-02-18T06:49:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7112355840 bytes 2026-02-18T06:49:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7159907328 bytes 2026-02-18T06:49:49.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7204647936 bytes 2026-02-18T06:49:49.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7230856192 bytes 2026-02-18T06:49:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7272769536 bytes 2026-02-18T06:49:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7346996224 bytes 2026-02-18T06:49:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7360160768 bytes 2026-02-18T06:49:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7454616576 bytes 2026-02-18T06:49:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7493275648 bytes 2026-02-18T06:49:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7550396416 bytes 2026-02-18T06:49:50.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7634191360 bytes 2026-02-18T06:49:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7658002432 bytes 2026-02-18T06:49:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7682693120 bytes 2026-02-18T06:49:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7708402688 bytes 2026-02-18T06:49:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7766915072 bytes 2026-02-18T06:49:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7780126720 bytes 2026-02-18T06:49:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7835229184 bytes 2026-02-18T06:49:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7919130624 bytes 2026-02-18T06:49:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7935085568 bytes 2026-02-18T06:49:50.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7993930752 bytes 2026-02-18T06:49:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8006931456 bytes 2026-02-18T06:49:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8014428160 bytes 2026-02-18T06:49:50.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8056513536 bytes 2026-02-18T06:49:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8133857280 bytes 2026-02-18T06:49:51.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8155259904 bytes 2026-02-18T06:49:51.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8158782464 bytes 2026-02-18T06:49:51.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8166617088 bytes 2026-02-18T06:49:51.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8233165824 bytes 2026-02-18T06:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8264407040 bytes 2026-02-18T06:49:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8325566464 bytes 2026-02-18T06:49:51.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8415252480 bytes 2026-02-18T06:49:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8455752704 bytes 2026-02-18T06:49:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8524439552 bytes 2026-02-18T06:49:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8578848768 bytes 2026-02-18T06:49:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8614162432 bytes 2026-02-18T06:49:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8652643328 bytes 2026-02-18T06:49:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8719009792 bytes 2026-02-18T06:49:51.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8752607232 bytes 2026-02-18T06:49:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8793776128 bytes 2026-02-18T06:49:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8805719040 bytes 2026-02-18T06:49:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 8824073216 bytes 2026-02-18T06:49:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:49:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_24 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:49:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_24 2026-02-18T06:49:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:49:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2026-02-18T06:49:52.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:49:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2026-02-18T06:49:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_24': 2026-02-18T06:49:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13a2d0e5caa2 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13a2d0e5caa2 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:52 2026 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:52 2026 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:52 2026 2026-02-18T06:49:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:49:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:49:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3756c8e9-5361-42c1-94d5-bb07eb3a568d 2026-02-18T06:49:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:49:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=8 2026-02-18T06:49:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:49:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:49:53.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 21011456 bytes 2026-02-18T06:49:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 51415040 bytes 2026-02-18T06:49:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 123952128 bytes 2026-02-18T06:49:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 149017600 bytes 2026-02-18T06:49:53.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 209588224 bytes 2026-02-18T06:49:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 232245248 bytes 2026-02-18T06:49:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 248229888 bytes 2026-02-18T06:49:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 319531008 bytes 2026-02-18T06:49:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 370075648 bytes 2026-02-18T06:49:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 391107584 bytes 2026-02-18T06:49:53.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 525059072 bytes 2026-02-18T06:49:53.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 536468480 bytes 2026-02-18T06:49:53.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 560235520 bytes 2026-02-18T06:49:53.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 588793856 bytes 2026-02-18T06:49:53.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 600719360 bytes 2026-02-18T06:49:53.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 617567232 bytes 2026-02-18T06:49:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 689175552 bytes 2026-02-18T06:49:54.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 719788032 bytes 2026-02-18T06:49:54.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 779459584 bytes 2026-02-18T06:49:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 817832960 bytes 2026-02-18T06:49:54.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 873410560 bytes 2026-02-18T06:49:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 940705792 bytes 2026-02-18T06:49:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1006197760 bytes 2026-02-18T06:49:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1041333248 bytes 2026-02-18T06:49:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1092272128 bytes 2026-02-18T06:49:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1175159808 bytes 2026-02-18T06:49:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1202238464 bytes 2026-02-18T06:49:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1224077312 bytes 2026-02-18T06:49:54.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1305145344 bytes 2026-02-18T06:49:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1355314176 bytes 2026-02-18T06:49:55.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1399503872 bytes 2026-02-18T06:49:55.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1419591680 bytes 2026-02-18T06:49:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1463616512 bytes 2026-02-18T06:49:55.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1510682624 bytes 2026-02-18T06:49:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1560857600 bytes 2026-02-18T06:49:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1616292864 bytes 2026-02-18T06:49:55.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1654455296 bytes 2026-02-18T06:49:55.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1683384320 bytes 2026-02-18T06:49:55.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1750693888 bytes 2026-02-18T06:49:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1803707392 bytes 2026-02-18T06:49:55.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1857172480 bytes 2026-02-18T06:49:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1910462464 bytes 2026-02-18T06:49:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1988857856 bytes 2026-02-18T06:49:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1989476352 bytes 2026-02-18T06:49:55.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2027707392 bytes 2026-02-18T06:49:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2096064512 bytes 2026-02-18T06:49:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2111776768 bytes 2026-02-18T06:49:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2144823296 bytes 2026-02-18T06:49:56.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2145424384 bytes 2026-02-18T06:49:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2178248704 bytes 2026-02-18T06:49:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2214413312 bytes 2026-02-18T06:49:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2274959360 bytes 2026-02-18T06:49:56.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2305624064 bytes 2026-02-18T06:49:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2370615296 bytes 2026-02-18T06:49:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2429891584 bytes 2026-02-18T06:49:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2500536320 bytes 2026-02-18T06:49:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2582456320 bytes 2026-02-18T06:49:56.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2682386432 bytes 2026-02-18T06:49:56.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2736826368 bytes 2026-02-18T06:49:57.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2861497344 bytes 2026-02-18T06:49:57.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2892621824 bytes 2026-02-18T06:49:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2894612480 bytes 2026-02-18T06:49:57.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2930876416 bytes 2026-02-18T06:49:57.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2954570752 bytes 2026-02-18T06:49:57.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3003157504 bytes 2026-02-18T06:49:57.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3040198656 bytes 2026-02-18T06:49:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3041546240 bytes 2026-02-18T06:49:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3071960064 bytes 2026-02-18T06:49:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3072577536 bytes 2026-02-18T06:49:57.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3144576000 bytes 2026-02-18T06:49:57.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3178421248 bytes 2026-02-18T06:49:57.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3193617408 bytes 2026-02-18T06:49:57.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3239458816 bytes 2026-02-18T06:49:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3281177600 bytes 2026-02-18T06:49:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3299192832 bytes 2026-02-18T06:49:58.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3368865792 bytes 2026-02-18T06:49:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3427306496 bytes 2026-02-18T06:49:58.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3474283520 bytes 2026-02-18T06:49:58.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3549757440 bytes 2026-02-18T06:49:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3598628864 bytes 2026-02-18T06:49:58.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3616799744 bytes 2026-02-18T06:49:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3670781952 bytes 2026-02-18T06:49:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3792807936 bytes 2026-02-18T06:49:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3816356864 bytes 2026-02-18T06:49:58.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3929812992 bytes 2026-02-18T06:49:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4017070080 bytes 2026-02-18T06:49:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4084978688 bytes 2026-02-18T06:49:59.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4137584640 bytes 2026-02-18T06:49:59.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4198566912 bytes 2026-02-18T06:49:59.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4260108288 bytes 2026-02-18T06:49:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4281683968 bytes 2026-02-18T06:49:59.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4306623488 bytes 2026-02-18T06:49:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4323515392 bytes 2026-02-18T06:49:59.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4348870656 bytes 2026-02-18T06:49:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4433496064 bytes 2026-02-18T06:49:59.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4510982144 bytes 2026-02-18T06:49:59.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4532206592 bytes 2026-02-18T06:49:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4602883072 bytes 2026-02-18T06:49:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4646760448 bytes 2026-02-18T06:49:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4687808512 bytes 2026-02-18T06:49:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4747944960 bytes 2026-02-18T06:50:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4769479680 bytes 2026-02-18T06:50:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4821017600 bytes 2026-02-18T06:50:00.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4866443264 bytes 2026-02-18T06:50:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4888967168 bytes 2026-02-18T06:50:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4970364928 bytes 2026-02-18T06:50:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5017768960 bytes 2026-02-18T06:50:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5094765568 bytes 2026-02-18T06:50:00.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5165213696 bytes 2026-02-18T06:50:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5207378944 bytes 2026-02-18T06:50:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5229037568 bytes 2026-02-18T06:50:00.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5305257984 bytes 2026-02-18T06:50:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5377612800 bytes 2026-02-18T06:50:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5438810112 bytes 2026-02-18T06:50:01.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5463242752 bytes 2026-02-18T06:50:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_25 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:50:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_25 2026-02-18T06:50:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2026-02-18T06:50:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:02.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_25': 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13ae5d8f2ad 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13ae5d8f2ad 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:01 2026 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:01 2026 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:01 2026 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b243d0f2-c374-431a-9dbd-c55ec3dfca56 2026-02-18T06:50:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:50:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=10 2026-02-18T06:50:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 53798912 bytes 2026-02-18T06:50:02.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 88211456 bytes 2026-02-18T06:50:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 174510080 bytes 2026-02-18T06:50:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 219432960 bytes 2026-02-18T06:50:02.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 349050880 bytes 2026-02-18T06:50:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 371544064 bytes 2026-02-18T06:50:02.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 372153344 bytes 2026-02-18T06:50:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 401122304 bytes 2026-02-18T06:50:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 502422528 bytes 2026-02-18T06:50:02.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 537624576 bytes 2026-02-18T06:50:03.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 593228800 bytes 2026-02-18T06:50:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 602887168 bytes 2026-02-18T06:50:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 640046080 bytes 2026-02-18T06:50:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 681958400 bytes 2026-02-18T06:50:03.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 696467456 bytes 2026-02-18T06:50:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 758955008 bytes 2026-02-18T06:50:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 814476288 bytes 2026-02-18T06:50:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 885060608 bytes 2026-02-18T06:50:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 890795008 bytes 2026-02-18T06:50:03.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 909799424 bytes 2026-02-18T06:50:03.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 983794688 bytes 2026-02-18T06:50:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1014603776 bytes 2026-02-18T06:50:03.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1070689280 bytes 2026-02-18T06:50:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1085049856 bytes 2026-02-18T06:50:03.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1141596160 bytes 2026-02-18T06:50:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1190433792 bytes 2026-02-18T06:50:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1243524096 bytes 2026-02-18T06:50:04.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1286697984 bytes 2026-02-18T06:50:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1346805760 bytes 2026-02-18T06:50:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1423465472 bytes 2026-02-18T06:50:04.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1424087040 bytes 2026-02-18T06:50:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1438908416 bytes 2026-02-18T06:50:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1498945536 bytes 2026-02-18T06:50:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1516765184 bytes 2026-02-18T06:50:04.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1522521088 bytes 2026-02-18T06:50:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1554861056 bytes 2026-02-18T06:50:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1597640704 bytes 2026-02-18T06:50:04.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1653180416 bytes 2026-02-18T06:50:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1687800832 bytes 2026-02-18T06:50:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1728131072 bytes 2026-02-18T06:50:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1790354432 bytes 2026-02-18T06:50:04.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1792986112 bytes 2026-02-18T06:50:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1831501824 bytes 2026-02-18T06:50:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1878573056 bytes 2026-02-18T06:50:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1902481408 bytes 2026-02-18T06:50:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1992306688 bytes 2026-02-18T06:50:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2033021952 bytes 2026-02-18T06:50:05.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2052379648 bytes 2026-02-18T06:50:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2062814208 bytes 2026-02-18T06:50:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2107402240 bytes 2026-02-18T06:50:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2192831488 bytes 2026-02-18T06:50:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2282649600 bytes 2026-02-18T06:50:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2315180032 bytes 2026-02-18T06:50:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2384367616 bytes 2026-02-18T06:50:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2436713472 bytes 2026-02-18T06:50:05.988 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:50:05.987+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbd0d825b0 2026-02-18T06:50:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2470292480 bytes 2026-02-18T06:50:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2476983296 bytes 2026-02-18T06:50:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2482660352 bytes 2026-02-18T06:50:06.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2504090624 bytes 2026-02-18T06:50:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2563884032 bytes 2026-02-18T06:50:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2603797504 bytes 2026-02-18T06:50:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2679553024 bytes 2026-02-18T06:50:06.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2733259776 bytes 2026-02-18T06:50:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2748844032 bytes 2026-02-18T06:50:06.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2792001536 bytes 2026-02-18T06:50:06.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2828812288 bytes 2026-02-18T06:50:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2859079680 bytes 2026-02-18T06:50:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2863295488 bytes 2026-02-18T06:50:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2961220608 bytes 2026-02-18T06:50:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2975521792 bytes 2026-02-18T06:50:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3093533696 bytes 2026-02-18T06:50:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3121268736 bytes 2026-02-18T06:50:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3132691456 bytes 2026-02-18T06:50:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3148590080 bytes 2026-02-18T06:50:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3226275840 bytes 2026-02-18T06:50:07.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3261106176 bytes 2026-02-18T06:50:07.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3294288896 bytes 2026-02-18T06:50:07.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3326321664 bytes 2026-02-18T06:50:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3337070592 bytes 2026-02-18T06:50:07.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3418755072 bytes 2026-02-18T06:50:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3461068800 bytes 2026-02-18T06:50:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3513168896 bytes 2026-02-18T06:50:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3539964928 bytes 2026-02-18T06:50:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3605558272 bytes 2026-02-18T06:50:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3650226176 bytes 2026-02-18T06:50:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3650861056 bytes 2026-02-18T06:50:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3654079488 bytes 2026-02-18T06:50:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3690642432 bytes 2026-02-18T06:50:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3731080192 bytes 2026-02-18T06:50:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3790974976 bytes 2026-02-18T06:50:08.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3856080896 bytes 2026-02-18T06:50:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3908452352 bytes 2026-02-18T06:50:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3972629504 bytes 2026-02-18T06:50:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4032733184 bytes 2026-02-18T06:50:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4062851072 bytes 2026-02-18T06:50:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4164341760 bytes 2026-02-18T06:50:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4251825152 bytes 2026-02-18T06:50:08.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4278767616 bytes 2026-02-18T06:50:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4307189760 bytes 2026-02-18T06:50:08.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4355400704 bytes 2026-02-18T06:50:08.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4411112448 bytes 2026-02-18T06:50:08.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4454334464 bytes 2026-02-18T06:50:09.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4475388928 bytes 2026-02-18T06:50:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4491529216 bytes 2026-02-18T06:50:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4562339840 bytes 2026-02-18T06:50:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4598734848 bytes 2026-02-18T06:50:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4644332544 bytes 2026-02-18T06:50:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4688709632 bytes 2026-02-18T06:50:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4785611776 bytes 2026-02-18T06:50:09.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4831470592 bytes 2026-02-18T06:50:09.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4888728576 bytes 2026-02-18T06:50:09.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4889329664 bytes 2026-02-18T06:50:09.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4889925632 bytes 2026-02-18T06:50:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4915837952 bytes 2026-02-18T06:50:09.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4946299904 bytes 2026-02-18T06:50:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4998335488 bytes 2026-02-18T06:50:09.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5073529856 bytes 2026-02-18T06:50:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5113161728 bytes 2026-02-18T06:50:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5152051200 bytes 2026-02-18T06:50:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5216561152 bytes 2026-02-18T06:50:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5263240192 bytes 2026-02-18T06:50:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5311091712 bytes 2026-02-18T06:50:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5432217600 bytes 2026-02-18T06:50:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5470453760 bytes 2026-02-18T06:50:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5527299072 bytes 2026-02-18T06:50:10.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5545035776 bytes 2026-02-18T06:50:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5607652352 bytes 2026-02-18T06:50:10.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5668347904 bytes 2026-02-18T06:50:10.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5696681984 bytes 2026-02-18T06:50:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5730548736 bytes 2026-02-18T06:50:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5821004800 bytes 2026-02-18T06:50:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5824031744 bytes 2026-02-18T06:50:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5887151104 bytes 2026-02-18T06:50:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5959115776 bytes 2026-02-18T06:50:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6033520640 bytes 2026-02-18T06:50:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6087962624 bytes 2026-02-18T06:50:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6172250112 bytes 2026-02-18T06:50:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6203954176 bytes 2026-02-18T06:50:11.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6232414208 bytes 2026-02-18T06:50:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6249834496 bytes 2026-02-18T06:50:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6329108480 bytes 2026-02-18T06:50:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6391529472 bytes 2026-02-18T06:50:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6415276032 bytes 2026-02-18T06:50:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6485714944 bytes 2026-02-18T06:50:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6511715328 bytes 2026-02-18T06:50:11.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6539381760 bytes 2026-02-18T06:50:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6557868032 bytes 2026-02-18T06:50:12.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:12.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:12.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_26 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:12.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_26 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:50:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2026-02-18T06:50:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_26': 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13ba73de535c 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13ba73de535c 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:12 2026 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:12 2026 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:12 2026 2026-02-18T06:50:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: bcbf3cf4-e8fd-4d05-8819-fa57991913f7 2026-02-18T06:50:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:50:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:50:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:13.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 53943296 bytes 2026-02-18T06:50:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 80201728 bytes 2026-02-18T06:50:13.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 107560960 bytes 2026-02-18T06:50:13.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 128246784 bytes 2026-02-18T06:50:13.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 128861184 bytes 2026-02-18T06:50:13.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 159841280 bytes 2026-02-18T06:50:13.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 185232384 bytes 2026-02-18T06:50:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 259273728 bytes 2026-02-18T06:50:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 297653248 bytes 2026-02-18T06:50:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 380070912 bytes 2026-02-18T06:50:13.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 391999488 bytes 2026-02-18T06:50:13.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 454200320 bytes 2026-02-18T06:50:13.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 479052800 bytes 2026-02-18T06:50:13.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 535258112 bytes 2026-02-18T06:50:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 577945600 bytes 2026-02-18T06:50:13.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 611585024 bytes 2026-02-18T06:50:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 657510400 bytes 2026-02-18T06:50:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 674952192 bytes 2026-02-18T06:50:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 694764544 bytes 2026-02-18T06:50:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 762610688 bytes 2026-02-18T06:50:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 775495680 bytes 2026-02-18T06:50:14.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 819169280 bytes 2026-02-18T06:50:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 897368064 bytes 2026-02-18T06:50:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 924811264 bytes 2026-02-18T06:50:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 935524352 bytes 2026-02-18T06:50:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 964932608 bytes 2026-02-18T06:50:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 998025216 bytes 2026-02-18T06:50:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1004219392 bytes 2026-02-18T06:50:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1075782656 bytes 2026-02-18T06:50:14.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1099158528 bytes 2026-02-18T06:50:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1190791168 bytes 2026-02-18T06:50:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1238511616 bytes 2026-02-18T06:50:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1265717248 bytes 2026-02-18T06:50:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1305532416 bytes 2026-02-18T06:50:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1370427392 bytes 2026-02-18T06:50:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1376941056 bytes 2026-02-18T06:50:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1409938432 bytes 2026-02-18T06:50:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1420917760 bytes 2026-02-18T06:50:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1461330944 bytes 2026-02-18T06:50:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1475301376 bytes 2026-02-18T06:50:15.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1515435008 bytes 2026-02-18T06:50:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1554621440 bytes 2026-02-18T06:50:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1601526784 bytes 2026-02-18T06:50:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1678612480 bytes 2026-02-18T06:50:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1726714880 bytes 2026-02-18T06:50:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1768838144 bytes 2026-02-18T06:50:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1823495168 bytes 2026-02-18T06:50:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1845457920 bytes 2026-02-18T06:50:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1916525568 bytes 2026-02-18T06:50:15.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1972797440 bytes 2026-02-18T06:50:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1991567360 bytes 2026-02-18T06:50:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2068969472 bytes 2026-02-18T06:50:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2149779456 bytes 2026-02-18T06:50:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2196244480 bytes 2026-02-18T06:50:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2246501376 bytes 2026-02-18T06:50:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2299308032 bytes 2026-02-18T06:50:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2312555520 bytes 2026-02-18T06:50:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2318862336 bytes 2026-02-18T06:50:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2392686592 bytes 2026-02-18T06:50:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2430938112 bytes 2026-02-18T06:50:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2479372288 bytes 2026-02-18T06:50:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2495922176 bytes 2026-02-18T06:50:16.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2571069440 bytes 2026-02-18T06:50:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2674981888 bytes 2026-02-18T06:50:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2754396160 bytes 2026-02-18T06:50:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2759313408 bytes 2026-02-18T06:50:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2825875456 bytes 2026-02-18T06:50:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2842618880 bytes 2026-02-18T06:50:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2929308672 bytes 2026-02-18T06:50:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2963474432 bytes 2026-02-18T06:50:17.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2976366592 bytes 2026-02-18T06:50:17.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2999311360 bytes 2026-02-18T06:50:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3042642944 bytes 2026-02-18T06:50:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3063318528 bytes 2026-02-18T06:50:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3109376000 bytes 2026-02-18T06:50:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3141095424 bytes 2026-02-18T06:50:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3224202240 bytes 2026-02-18T06:50:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3327102976 bytes 2026-02-18T06:50:17.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3351149568 bytes 2026-02-18T06:50:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3357093888 bytes 2026-02-18T06:50:17.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3390730240 bytes 2026-02-18T06:50:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3416505344 bytes 2026-02-18T06:50:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3439736832 bytes 2026-02-18T06:50:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3469233152 bytes 2026-02-18T06:50:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3495473152 bytes 2026-02-18T06:50:18.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_27 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_27 2026-02-18T06:50:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:18.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2026-02-18T06:50:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_27': 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13c6e3858cf7 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13c6e3858cf7 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:18 2026 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:18 2026 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:18 2026 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: f601ab61-b7bf-4d56-836f-81dda8d76269 2026-02-18T06:50:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:50:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:50:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 59959296 bytes 2026-02-18T06:50:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 84923392 bytes 2026-02-18T06:50:19.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 150063104 bytes 2026-02-18T06:50:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 207928320 bytes 2026-02-18T06:50:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 243870720 bytes 2026-02-18T06:50:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 276799488 bytes 2026-02-18T06:50:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 392747008 bytes 2026-02-18T06:50:19.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 424225792 bytes 2026-02-18T06:50:19.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 468701184 bytes 2026-02-18T06:50:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 510913536 bytes 2026-02-18T06:50:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 557091840 bytes 2026-02-18T06:50:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 593489920 bytes 2026-02-18T06:50:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 631261184 bytes 2026-02-18T06:50:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 681534464 bytes 2026-02-18T06:50:20.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 775303168 bytes 2026-02-18T06:50:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 815533056 bytes 2026-02-18T06:50:20.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 829735936 bytes 2026-02-18T06:50:20.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 864731136 bytes 2026-02-18T06:50:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 927924224 bytes 2026-02-18T06:50:20.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1020793856 bytes 2026-02-18T06:50:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1060335616 bytes 2026-02-18T06:50:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1107173376 bytes 2026-02-18T06:50:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1189566464 bytes 2026-02-18T06:50:20.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1214149632 bytes 2026-02-18T06:50:20.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1248743424 bytes 2026-02-18T06:50:20.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1300487168 bytes 2026-02-18T06:50:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1343888384 bytes 2026-02-18T06:50:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1388324864 bytes 2026-02-18T06:50:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1466907648 bytes 2026-02-18T06:50:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1522617344 bytes 2026-02-18T06:50:21.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1586305024 bytes 2026-02-18T06:50:21.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1692702720 bytes 2026-02-18T06:50:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1730326528 bytes 2026-02-18T06:50:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1757592576 bytes 2026-02-18T06:50:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1811117056 bytes 2026-02-18T06:50:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1866407936 bytes 2026-02-18T06:50:21.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1910300672 bytes 2026-02-18T06:50:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1944307712 bytes 2026-02-18T06:50:21.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1966097408 bytes 2026-02-18T06:50:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2011778048 bytes 2026-02-18T06:50:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2108788736 bytes 2026-02-18T06:50:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2135859200 bytes 2026-02-18T06:50:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2150558720 bytes 2026-02-18T06:50:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2172913664 bytes 2026-02-18T06:50:22.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2226478080 bytes 2026-02-18T06:50:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2261435392 bytes 2026-02-18T06:50:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2324265984 bytes 2026-02-18T06:50:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2357721088 bytes 2026-02-18T06:50:22.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2410303488 bytes 2026-02-18T06:50:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2492534784 bytes 2026-02-18T06:50:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2505013248 bytes 2026-02-18T06:50:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2557083648 bytes 2026-02-18T06:50:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2562790400 bytes 2026-02-18T06:50:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2565573632 bytes 2026-02-18T06:50:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2644404224 bytes 2026-02-18T06:50:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2682284032 bytes 2026-02-18T06:50:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2718961664 bytes 2026-02-18T06:50:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2770525184 bytes 2026-02-18T06:50:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2774677504 bytes 2026-02-18T06:50:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2793472000 bytes 2026-02-18T06:50:23.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2830803968 bytes 2026-02-18T06:50:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2839830528 bytes 2026-02-18T06:50:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2885731328 bytes 2026-02-18T06:50:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2910364672 bytes 2026-02-18T06:50:23.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3009444864 bytes 2026-02-18T06:50:23.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3087067136 bytes 2026-02-18T06:50:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3106290688 bytes 2026-02-18T06:50:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3124746240 bytes 2026-02-18T06:50:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3204004864 bytes 2026-02-18T06:50:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3225069568 bytes 2026-02-18T06:50:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3266189312 bytes 2026-02-18T06:50:23.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3327150080 bytes 2026-02-18T06:50:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3383128064 bytes 2026-02-18T06:50:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3388184576 bytes 2026-02-18T06:50:24.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3458427904 bytes 2026-02-18T06:50:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3467070464 bytes 2026-02-18T06:50:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3500916736 bytes 2026-02-18T06:50:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3529396224 bytes 2026-02-18T06:50:24.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3577955328 bytes 2026-02-18T06:50:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3611750400 bytes 2026-02-18T06:50:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3667773440 bytes 2026-02-18T06:50:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3706103808 bytes 2026-02-18T06:50:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3706708992 bytes 2026-02-18T06:50:24.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3715210240 bytes 2026-02-18T06:50:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3774123008 bytes 2026-02-18T06:50:24.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3846790144 bytes 2026-02-18T06:50:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3877207040 bytes 2026-02-18T06:50:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3882509312 bytes 2026-02-18T06:50:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3989347328 bytes 2026-02-18T06:50:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4034876416 bytes 2026-02-18T06:50:24.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.42860597720978544) > weight (0.25) 2026-02-18T06:50:24.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.929926706759616) > weight (0.25) 2026-02-18T06:50:24.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5708256248049901) > weight (0.25) 2026-02-18T06:50:24.934 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.784131189653186) > weight (0.25) 2026-02-18T06:50:24.934 INFO:tasks.rbd_mirror_thrash:waiting for 119.84040189400453 secs before thrashing 2026-02-18T06:50:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4098894848 bytes 2026-02-18T06:50:25.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4123367424 bytes 2026-02-18T06:50:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4203195392 bytes 2026-02-18T06:50:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4203822080 bytes 2026-02-18T06:50:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4251328512 bytes 2026-02-18T06:50:25.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4287111168 bytes 2026-02-18T06:50:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4316100608 bytes 2026-02-18T06:50:25.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4365162496 bytes 2026-02-18T06:50:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4425137152 bytes 2026-02-18T06:50:25.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4466086912 bytes 2026-02-18T06:50:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4466671616 bytes 2026-02-18T06:50:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4504481792 bytes 2026-02-18T06:50:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4537108480 bytes 2026-02-18T06:50:25.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4613310464 bytes 2026-02-18T06:50:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4651450368 bytes 2026-02-18T06:50:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4660277248 bytes 2026-02-18T06:50:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4695673856 bytes 2026-02-18T06:50:25.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4737557504 bytes 2026-02-18T06:50:25.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4738466816 bytes 2026-02-18T06:50:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4783802368 bytes 2026-02-18T06:50:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_28 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:26.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_28 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:50:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2026-02-18T06:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2026-02-18T06:50:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_28': 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13d2265f884d 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13d2265f884d 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:26 2026 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:26 2026 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:26 2026 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 87cd4a83-4564-4b99-b3f5-5b048052b5e2 2026-02-18T06:50:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-18T06:50:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:50:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:50:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:27.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 8965120 bytes 2026-02-18T06:50:27.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 27249664 bytes 2026-02-18T06:50:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 56743936 bytes 2026-02-18T06:50:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 163975168 bytes 2026-02-18T06:50:27.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 221444096 bytes 2026-02-18T06:50:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 246772736 bytes 2026-02-18T06:50:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 274827264 bytes 2026-02-18T06:50:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 287963136 bytes 2026-02-18T06:50:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 339998720 bytes 2026-02-18T06:50:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 395609088 bytes 2026-02-18T06:50:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 445716480 bytes 2026-02-18T06:50:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 474697728 bytes 2026-02-18T06:50:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 542020608 bytes 2026-02-18T06:50:27.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 594992128 bytes 2026-02-18T06:50:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 621096960 bytes 2026-02-18T06:50:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 669742080 bytes 2026-02-18T06:50:28.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 707799040 bytes 2026-02-18T06:50:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 733912064 bytes 2026-02-18T06:50:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 814514176 bytes 2026-02-18T06:50:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 854692864 bytes 2026-02-18T06:50:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 874309632 bytes 2026-02-18T06:50:28.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 891752448 bytes 2026-02-18T06:50:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 943844352 bytes 2026-02-18T06:50:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 987246592 bytes 2026-02-18T06:50:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1026987008 bytes 2026-02-18T06:50:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1038559232 bytes 2026-02-18T06:50:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1074051072 bytes 2026-02-18T06:50:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1116527616 bytes 2026-02-18T06:50:28.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1138061312 bytes 2026-02-18T06:50:28.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1188756480 bytes 2026-02-18T06:50:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1239318528 bytes 2026-02-18T06:50:28.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1287359488 bytes 2026-02-18T06:50:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1321913344 bytes 2026-02-18T06:50:29.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1361193984 bytes 2026-02-18T06:50:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1361810432 bytes 2026-02-18T06:50:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1394295808 bytes 2026-02-18T06:50:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1485751296 bytes 2026-02-18T06:50:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1486386176 bytes 2026-02-18T06:50:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1519380480 bytes 2026-02-18T06:50:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1524993024 bytes 2026-02-18T06:50:29.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1551412224 bytes 2026-02-18T06:50:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1566811136 bytes 2026-02-18T06:50:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1593614336 bytes 2026-02-18T06:50:29.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1605154816 bytes 2026-02-18T06:50:29.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1660473344 bytes 2026-02-18T06:50:29.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1696582656 bytes 2026-02-18T06:50:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1697216512 bytes 2026-02-18T06:50:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1705137152 bytes 2026-02-18T06:50:29.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1739912192 bytes 2026-02-18T06:50:29.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1753693184 bytes 2026-02-18T06:50:29.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1777756160 bytes 2026-02-18T06:50:29.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1840890880 bytes 2026-02-18T06:50:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1870925824 bytes 2026-02-18T06:50:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1954418688 bytes 2026-02-18T06:50:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2000196608 bytes 2026-02-18T06:50:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2096048128 bytes 2026-02-18T06:50:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2144812032 bytes 2026-02-18T06:50:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2189943808 bytes 2026-02-18T06:50:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2201504768 bytes 2026-02-18T06:50:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2270408704 bytes 2026-02-18T06:50:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2334884864 bytes 2026-02-18T06:50:30.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2345647104 bytes 2026-02-18T06:50:30.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2353084416 bytes 2026-02-18T06:50:30.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2384046080 bytes 2026-02-18T06:50:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2435678208 bytes 2026-02-18T06:50:30.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2467073024 bytes 2026-02-18T06:50:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2475710464 bytes 2026-02-18T06:50:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2487355392 bytes 2026-02-18T06:50:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2523859968 bytes 2026-02-18T06:50:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2555506688 bytes 2026-02-18T06:50:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2566466560 bytes 2026-02-18T06:50:31.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2581135360 bytes 2026-02-18T06:50:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2639056896 bytes 2026-02-18T06:50:31.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2676950016 bytes 2026-02-18T06:50:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2734993408 bytes 2026-02-18T06:50:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2741876736 bytes 2026-02-18T06:50:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2779891712 bytes 2026-02-18T06:50:31.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2811031552 bytes 2026-02-18T06:50:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2868427776 bytes 2026-02-18T06:50:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2894154752 bytes 2026-02-18T06:50:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 2908290048 bytes 2026-02-18T06:50:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 2934154240 bytes 2026-02-18T06:50:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 2934758400 bytes 2026-02-18T06:50:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 2974439424 bytes 2026-02-18T06:50:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 2984708096 bytes 2026-02-18T06:50:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3066701824 bytes 2026-02-18T06:50:31.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3093650432 bytes 2026-02-18T06:50:31.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3115089920 bytes 2026-02-18T06:50:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3195798528 bytes 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_29 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-18T06:50:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_29 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2026-02-18T06:50:33.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_29': 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13de263049e1 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13de263049e1 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:32 2026 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:32 2026 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:32 2026 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d01291bd-af69-4270-aa28-f3563eed7534 2026-02-18T06:50:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:50:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:50:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 66381824 bytes 2026-02-18T06:50:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 81632256 bytes 2026-02-18T06:50:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 115354624 bytes 2026-02-18T06:50:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 139006976 bytes 2026-02-18T06:50:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 199522304 bytes 2026-02-18T06:50:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 261143552 bytes 2026-02-18T06:50:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 350191616 bytes 2026-02-18T06:50:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 433388544 bytes 2026-02-18T06:50:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 483879936 bytes 2026-02-18T06:50:33.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 554544128 bytes 2026-02-18T06:50:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 563654656 bytes 2026-02-18T06:50:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 596055040 bytes 2026-02-18T06:50:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 632587264 bytes 2026-02-18T06:50:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 678263808 bytes 2026-02-18T06:50:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 714005504 bytes 2026-02-18T06:50:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 738895872 bytes 2026-02-18T06:50:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 743134208 bytes 2026-02-18T06:50:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 809327616 bytes 2026-02-18T06:50:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 840964096 bytes 2026-02-18T06:50:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 872953856 bytes 2026-02-18T06:50:34.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 963055616 bytes 2026-02-18T06:50:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1014267904 bytes 2026-02-18T06:50:34.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1083460608 bytes 2026-02-18T06:50:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1161940992 bytes 2026-02-18T06:50:34.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1202927616 bytes 2026-02-18T06:50:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1264684032 bytes 2026-02-18T06:50:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1338887168 bytes 2026-02-18T06:50:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1420375040 bytes 2026-02-18T06:50:34.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1493591040 bytes 2026-02-18T06:50:35.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1509827584 bytes 2026-02-18T06:50:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1592587264 bytes 2026-02-18T06:50:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1656816640 bytes 2026-02-18T06:50:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1674800128 bytes 2026-02-18T06:50:35.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1742102528 bytes 2026-02-18T06:50:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1742709760 bytes 2026-02-18T06:50:35.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1770454016 bytes 2026-02-18T06:50:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1795435520 bytes 2026-02-18T06:50:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1847900160 bytes 2026-02-18T06:50:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1878469632 bytes 2026-02-18T06:50:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1937989632 bytes 2026-02-18T06:50:35.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1963582464 bytes 2026-02-18T06:50:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1976070144 bytes 2026-02-18T06:50:35.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1995015168 bytes 2026-02-18T06:50:35.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2014891008 bytes 2026-02-18T06:50:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2121063424 bytes 2026-02-18T06:50:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2144594944 bytes 2026-02-18T06:50:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2234777600 bytes 2026-02-18T06:50:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2257847296 bytes 2026-02-18T06:50:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2312621056 bytes 2026-02-18T06:50:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2313243648 bytes 2026-02-18T06:50:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2400658432 bytes 2026-02-18T06:50:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2447931392 bytes 2026-02-18T06:50:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2488774656 bytes 2026-02-18T06:50:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2526126080 bytes 2026-02-18T06:50:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2622857216 bytes 2026-02-18T06:50:36.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2695054336 bytes 2026-02-18T06:50:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2706560000 bytes 2026-02-18T06:50:36.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2775289856 bytes 2026-02-18T06:50:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2800521216 bytes 2026-02-18T06:50:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2872476672 bytes 2026-02-18T06:50:36.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2892937216 bytes 2026-02-18T06:50:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2962401280 bytes 2026-02-18T06:50:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 3007594496 bytes 2026-02-18T06:50:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3028882432 bytes 2026-02-18T06:50:37.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3123411968 bytes 2026-02-18T06:50:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3175101440 bytes 2026-02-18T06:50:37.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3227938816 bytes 2026-02-18T06:50:37.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3238638592 bytes 2026-02-18T06:50:37.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3275934720 bytes 2026-02-18T06:50:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3293203456 bytes 2026-02-18T06:50:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3355066368 bytes 2026-02-18T06:50:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3383545856 bytes 2026-02-18T06:50:37.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3438020608 bytes 2026-02-18T06:50:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3479032832 bytes 2026-02-18T06:50:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3496098816 bytes 2026-02-18T06:50:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3538614272 bytes 2026-02-18T06:50:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3582149632 bytes 2026-02-18T06:50:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3656140800 bytes 2026-02-18T06:50:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3714847744 bytes 2026-02-18T06:50:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3745258496 bytes 2026-02-18T06:50:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3823757312 bytes 2026-02-18T06:50:38.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3871887360 bytes 2026-02-18T06:50:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3872493568 bytes 2026-02-18T06:50:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3939899392 bytes 2026-02-18T06:50:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 4004576256 bytes 2026-02-18T06:50:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4026373120 bytes 2026-02-18T06:50:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4034400256 bytes 2026-02-18T06:50:38.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4056989696 bytes 2026-02-18T06:50:38.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4088722432 bytes 2026-02-18T06:50:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4120226816 bytes 2026-02-18T06:50:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4167103488 bytes 2026-02-18T06:50:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4217744384 bytes 2026-02-18T06:50:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4277803008 bytes 2026-02-18T06:50:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4327902208 bytes 2026-02-18T06:50:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4345868288 bytes 2026-02-18T06:50:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4414994432 bytes 2026-02-18T06:50:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4453386240 bytes 2026-02-18T06:50:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4465433600 bytes 2026-02-18T06:50:39.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4516614144 bytes 2026-02-18T06:50:39.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4606466048 bytes 2026-02-18T06:50:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4641422336 bytes 2026-02-18T06:50:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4681265152 bytes 2026-02-18T06:50:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4700920832 bytes 2026-02-18T06:50:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4766703616 bytes 2026-02-18T06:50:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4823265280 bytes 2026-02-18T06:50:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4865553408 bytes 2026-02-18T06:50:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4925433856 bytes 2026-02-18T06:50:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4972707840 bytes 2026-02-18T06:50:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4988267520 bytes 2026-02-18T06:50:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4988872704 bytes 2026-02-18T06:50:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5047141376 bytes 2026-02-18T06:50:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5077293056 bytes 2026-02-18T06:50:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5161812992 bytes 2026-02-18T06:50:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5227794432 bytes 2026-02-18T06:50:40.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5273987072 bytes 2026-02-18T06:50:40.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5285284864 bytes 2026-02-18T06:50:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5299507200 bytes 2026-02-18T06:50:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5359115264 bytes 2026-02-18T06:50:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5362204672 bytes 2026-02-18T06:50:40.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5382039552 bytes 2026-02-18T06:50:40.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9460998302846038) > weight (0.25) 2026-02-18T06:50:40.645 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8839260457142639) > weight (0.25) 2026-02-18T06:50:40.645 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T06:50:40.645 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T06:50:40.645 INFO:tasks.rbd_mirror_thrash:waiting for 4.244230428907278 secs before reviving daemons 2026-02-18T06:50:40.646 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:50:40.643+0000 7f5fa4d98640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 40081) UID: 1000 2026-02-18T06:50:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5450992640 bytes 2026-02-18T06:50:40.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5486102528 bytes 2026-02-18T06:50:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5499203584 bytes 2026-02-18T06:50:40.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5551291392 bytes 2026-02-18T06:50:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5593663488 bytes 2026-02-18T06:50:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5675250688 bytes 2026-02-18T06:50:41.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5721284608 bytes 2026-02-18T06:50:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5773444096 bytes 2026-02-18T06:50:41.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5799012352 bytes 2026-02-18T06:50:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5824386048 bytes 2026-02-18T06:50:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5882401792 bytes 2026-02-18T06:50:41.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5916167168 bytes 2026-02-18T06:50:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6037927936 bytes 2026-02-18T06:50:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6112911360 bytes 2026-02-18T06:50:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6141105152 bytes 2026-02-18T06:50:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6194574336 bytes 2026-02-18T06:50:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6247501824 bytes 2026-02-18T06:50:41.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6263780352 bytes 2026-02-18T06:50:41.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6296100864 bytes 2026-02-18T06:50:41.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6330795008 bytes 2026-02-18T06:50:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6372456448 bytes 2026-02-18T06:50:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6394658816 bytes 2026-02-18T06:50:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6417187840 bytes 2026-02-18T06:50:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6482894848 bytes 2026-02-18T06:50:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6511395840 bytes 2026-02-18T06:50:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6570500096 bytes 2026-02-18T06:50:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6651945984 bytes 2026-02-18T06:50:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6695686144 bytes 2026-02-18T06:50:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6762569728 bytes 2026-02-18T06:50:42.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6805322752 bytes 2026-02-18T06:50:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6817894400 bytes 2026-02-18T06:50:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6844035072 bytes 2026-02-18T06:50:42.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6859732992 bytes 2026-02-18T06:50:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6950642688 bytes 2026-02-18T06:50:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6980446208 bytes 2026-02-18T06:50:42.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 7008474112 bytes 2026-02-18T06:50:42.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7063947264 bytes 2026-02-18T06:50:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7102267392 bytes 2026-02-18T06:50:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7116368896 bytes 2026-02-18T06:50:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7173056512 bytes 2026-02-18T06:50:43.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7204747264 bytes 2026-02-18T06:50:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7248912384 bytes 2026-02-18T06:50:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7273694208 bytes 2026-02-18T06:50:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7335954432 bytes 2026-02-18T06:50:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7360486400 bytes 2026-02-18T06:50:43.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7361102848 bytes 2026-02-18T06:50:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7384105984 bytes 2026-02-18T06:50:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7441128448 bytes 2026-02-18T06:50:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7493712896 bytes 2026-02-18T06:50:43.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7525086208 bytes 2026-02-18T06:50:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7544609792 bytes 2026-02-18T06:50:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7545205760 bytes 2026-02-18T06:50:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7588469760 bytes 2026-02-18T06:50:43.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7659545600 bytes 2026-02-18T06:50:43.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7664113664 bytes 2026-02-18T06:50:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7672418304 bytes 2026-02-18T06:50:43.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7708164096 bytes 2026-02-18T06:50:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_30 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_30 2026-02-18T06:50:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:44.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2026-02-18T06:50:44.890 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T06:50:44.891 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:50:44.891 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T06:50:44.891 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T06:50:44.892 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T06:50:44.895 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T06:50:44.895 INFO:tasks.rbd_mirror_thrash:waiting for 66.68654348592206 secs before thrashing 2026-02-18T06:50:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2026-02-18T06:50:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_30': 2026-02-18T06:50:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13ea692c513 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13ea692c513 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:44 2026 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:44 2026 2026-02-18T06:50:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:44 2026 2026-02-18T06:50:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 8ba02ff9-8f1b-476e-a125-0499e1932593 2026-02-18T06:50:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:50:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:45.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:50:45.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:45.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 28477440 bytes 2026-02-18T06:50:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 105196544 bytes 2026-02-18T06:50:45.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 206044160 bytes 2026-02-18T06:50:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 327111680 bytes 2026-02-18T06:50:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 346387456 bytes 2026-02-18T06:50:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 410055680 bytes 2026-02-18T06:50:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 438200320 bytes 2026-02-18T06:50:45.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 438814720 bytes 2026-02-18T06:50:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 443331584 bytes 2026-02-18T06:50:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 457912320 bytes 2026-02-18T06:50:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 477830144 bytes 2026-02-18T06:50:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 495448064 bytes 2026-02-18T06:50:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 527816704 bytes 2026-02-18T06:50:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 564353024 bytes 2026-02-18T06:50:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 649035776 bytes 2026-02-18T06:50:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 699465728 bytes 2026-02-18T06:50:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 730743808 bytes 2026-02-18T06:50:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 774975488 bytes 2026-02-18T06:50:46.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 819050496 bytes 2026-02-18T06:50:46.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 852007936 bytes 2026-02-18T06:50:46.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 858177536 bytes 2026-02-18T06:50:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 885150720 bytes 2026-02-18T06:50:46.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 937563136 bytes 2026-02-18T06:50:46.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 973373440 bytes 2026-02-18T06:50:46.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 985281536 bytes 2026-02-18T06:50:46.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1030666240 bytes 2026-02-18T06:50:46.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1031281664 bytes 2026-02-18T06:50:46.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1093022720 bytes 2026-02-18T06:50:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1140285440 bytes 2026-02-18T06:50:46.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1181057024 bytes 2026-02-18T06:50:46.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1225023488 bytes 2026-02-18T06:50:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1277549568 bytes 2026-02-18T06:50:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1329959936 bytes 2026-02-18T06:50:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1415851008 bytes 2026-02-18T06:50:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1482687488 bytes 2026-02-18T06:50:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1536944128 bytes 2026-02-18T06:50:47.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1558768640 bytes 2026-02-18T06:50:47.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1575852032 bytes 2026-02-18T06:50:47.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1631722496 bytes 2026-02-18T06:50:47.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1713523712 bytes 2026-02-18T06:50:47.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1753668608 bytes 2026-02-18T06:50:47.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1810723840 bytes 2026-02-18T06:50:47.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1831962624 bytes 2026-02-18T06:50:47.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1901000704 bytes 2026-02-18T06:50:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1901814784 bytes 2026-02-18T06:50:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1966180352 bytes 2026-02-18T06:50:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1988934656 bytes 2026-02-18T06:50:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2031651840 bytes 2026-02-18T06:50:48.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2057037824 bytes 2026-02-18T06:50:48.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2127939584 bytes 2026-02-18T06:50:48.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2189726720 bytes 2026-02-18T06:50:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2207717376 bytes 2026-02-18T06:50:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2305768448 bytes 2026-02-18T06:50:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2346868736 bytes 2026-02-18T06:50:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2398998528 bytes 2026-02-18T06:50:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2437315584 bytes 2026-02-18T06:50:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2476645376 bytes 2026-02-18T06:50:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2501229568 bytes 2026-02-18T06:50:48.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2610173952 bytes 2026-02-18T06:50:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2639356928 bytes 2026-02-18T06:50:49.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2658095104 bytes 2026-02-18T06:50:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2705154048 bytes 2026-02-18T06:50:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2709872640 bytes 2026-02-18T06:50:49.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2765566976 bytes 2026-02-18T06:50:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2791916544 bytes 2026-02-18T06:50:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2844211200 bytes 2026-02-18T06:50:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2858494976 bytes 2026-02-18T06:50:49.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2867575808 bytes 2026-02-18T06:50:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2896005120 bytes 2026-02-18T06:50:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2925423616 bytes 2026-02-18T06:50:49.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2956492800 bytes 2026-02-18T06:50:49.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3007939584 bytes 2026-02-18T06:50:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3017656320 bytes 2026-02-18T06:50:49.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3044335616 bytes 2026-02-18T06:50:49.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3077709824 bytes 2026-02-18T06:50:49.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3119675392 bytes 2026-02-18T06:50:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3240113152 bytes 2026-02-18T06:50:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3309392896 bytes 2026-02-18T06:50:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3349409792 bytes 2026-02-18T06:50:50.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3385119744 bytes 2026-02-18T06:50:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3414452224 bytes 2026-02-18T06:50:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3448397824 bytes 2026-02-18T06:50:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3476723712 bytes 2026-02-18T06:50:50.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3477526528 bytes 2026-02-18T06:50:50.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3514656768 bytes 2026-02-18T06:50:50.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3519247360 bytes 2026-02-18T06:50:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3590985728 bytes 2026-02-18T06:50:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3645719552 bytes 2026-02-18T06:50:50.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3676890112 bytes 2026-02-18T06:50:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3698826240 bytes 2026-02-18T06:50:50.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3712723968 bytes 2026-02-18T06:50:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3760336896 bytes 2026-02-18T06:50:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3778851840 bytes 2026-02-18T06:50:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3838747648 bytes 2026-02-18T06:50:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3848117248 bytes 2026-02-18T06:50:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3902576640 bytes 2026-02-18T06:50:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3903186944 bytes 2026-02-18T06:50:51.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3961789440 bytes 2026-02-18T06:50:51.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3999612928 bytes 2026-02-18T06:50:51.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4078501888 bytes 2026-02-18T06:50:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4117398528 bytes 2026-02-18T06:50:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4240645120 bytes 2026-02-18T06:50:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4250836992 bytes 2026-02-18T06:50:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4290724864 bytes 2026-02-18T06:50:51.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4354903040 bytes 2026-02-18T06:50:51.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4376558592 bytes 2026-02-18T06:50:51.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4389226496 bytes 2026-02-18T06:50:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4443234304 bytes 2026-02-18T06:50:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4497053696 bytes 2026-02-18T06:50:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4517412864 bytes 2026-02-18T06:50:51.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4601819136 bytes 2026-02-18T06:50:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4604508160 bytes 2026-02-18T06:50:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4680478720 bytes 2026-02-18T06:50:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4724547584 bytes 2026-02-18T06:50:52.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4776725504 bytes 2026-02-18T06:50:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4810000384 bytes 2026-02-18T06:50:52.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4868593664 bytes 2026-02-18T06:50:52.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4931431424 bytes 2026-02-18T06:50:52.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4953270272 bytes 2026-02-18T06:50:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4988573696 bytes 2026-02-18T06:50:52.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4996200448 bytes 2026-02-18T06:50:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5031728128 bytes 2026-02-18T06:50:52.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5100899328 bytes 2026-02-18T06:50:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5149455360 bytes 2026-02-18T06:50:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5166137344 bytes 2026-02-18T06:50:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5214713856 bytes 2026-02-18T06:50:52.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5297651712 bytes 2026-02-18T06:50:53.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5380721664 bytes 2026-02-18T06:50:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5431250944 bytes 2026-02-18T06:50:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5505766400 bytes 2026-02-18T06:50:53.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5530548224 bytes 2026-02-18T06:50:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5603938304 bytes 2026-02-18T06:50:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5652675584 bytes 2026-02-18T06:50:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5739495424 bytes 2026-02-18T06:50:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5765605376 bytes 2026-02-18T06:50:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5848599552 bytes 2026-02-18T06:50:53.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5912025088 bytes 2026-02-18T06:50:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5967394816 bytes 2026-02-18T06:50:53.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6009907200 bytes 2026-02-18T06:50:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6085196800 bytes 2026-02-18T06:50:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6163646464 bytes 2026-02-18T06:50:54.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6195937280 bytes 2026-02-18T06:50:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6207161344 bytes 2026-02-18T06:50:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6269424640 bytes 2026-02-18T06:50:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6308389888 bytes 2026-02-18T06:50:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6344099840 bytes 2026-02-18T06:50:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6379450368 bytes 2026-02-18T06:50:54.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6398595072 bytes 2026-02-18T06:50:54.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6423453696 bytes 2026-02-18T06:50:54.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6470817792 bytes 2026-02-18T06:50:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6495500288 bytes 2026-02-18T06:50:54.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6538636288 bytes 2026-02-18T06:50:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6539237376 bytes 2026-02-18T06:50:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6595419136 bytes 2026-02-18T06:50:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6630875136 bytes 2026-02-18T06:50:55.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6682923008 bytes 2026-02-18T06:50:55.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6734411776 bytes 2026-02-18T06:50:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6756684800 bytes 2026-02-18T06:50:55.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6798990336 bytes 2026-02-18T06:50:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6851638272 bytes 2026-02-18T06:50:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6976392192 bytes 2026-02-18T06:50:55.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6993953792 bytes 2026-02-18T06:50:55.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7030759424 bytes 2026-02-18T06:50:55.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7060763648 bytes 2026-02-18T06:50:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7122725888 bytes 2026-02-18T06:50:55.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7177644032 bytes 2026-02-18T06:50:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7257865216 bytes 2026-02-18T06:50:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7294728192 bytes 2026-02-18T06:50:55.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7321451520 bytes 2026-02-18T06:50:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7322063872 bytes 2026-02-18T06:50:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7346228224 bytes 2026-02-18T06:50:56.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7384393728 bytes 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_31 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2026-02-18T06:50:56.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:50:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_31 2026-02-18T06:50:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:50:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:50:56.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2026-02-18T06:50:57.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:50:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2026-02-18T06:50:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_31': 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 13fbb3e50d7f 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.13fbb3e50d7f 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:50:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:56 2026 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:56 2026 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:56 2026 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 68ddba5c-0c32-4a47-9248-13cbd388326b 2026-02-18T06:50:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:50:57.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:50:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=10 2026-02-18T06:50:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:50:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:50:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 13519872 bytes 2026-02-18T06:50:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 54076416 bytes 2026-02-18T06:50:57.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 80122880 bytes 2026-02-18T06:50:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 118273024 bytes 2026-02-18T06:50:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 120898560 bytes 2026-02-18T06:50:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 200817664 bytes 2026-02-18T06:50:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 300308480 bytes 2026-02-18T06:50:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 329226240 bytes 2026-02-18T06:50:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 366212096 bytes 2026-02-18T06:50:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 388396032 bytes 2026-02-18T06:50:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 410573824 bytes 2026-02-18T06:50:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 452743168 bytes 2026-02-18T06:50:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 453370880 bytes 2026-02-18T06:50:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 517673984 bytes 2026-02-18T06:50:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 571524096 bytes 2026-02-18T06:50:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 585465856 bytes 2026-02-18T06:50:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 622526464 bytes 2026-02-18T06:50:58.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 668869632 bytes 2026-02-18T06:50:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 726794240 bytes 2026-02-18T06:50:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 769989632 bytes 2026-02-18T06:50:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 813854720 bytes 2026-02-18T06:50:58.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 872679424 bytes 2026-02-18T06:50:58.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 910777344 bytes 2026-02-18T06:50:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 948113408 bytes 2026-02-18T06:50:58.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 972143616 bytes 2026-02-18T06:50:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 977651712 bytes 2026-02-18T06:50:58.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1004141568 bytes 2026-02-18T06:50:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1031753728 bytes 2026-02-18T06:50:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1140762624 bytes 2026-02-18T06:50:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1198758912 bytes 2026-02-18T06:50:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1214411776 bytes 2026-02-18T06:50:58.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1227873280 bytes 2026-02-18T06:50:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1269954560 bytes 2026-02-18T06:50:58.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1339033600 bytes 2026-02-18T06:50:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1385684992 bytes 2026-02-18T06:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1411159040 bytes 2026-02-18T06:50:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1448440832 bytes 2026-02-18T06:50:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1525773312 bytes 2026-02-18T06:50:59.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1550872576 bytes 2026-02-18T06:50:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1579332608 bytes 2026-02-18T06:50:59.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1614718976 bytes 2026-02-18T06:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1629365248 bytes 2026-02-18T06:50:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1681992704 bytes 2026-02-18T06:50:59.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1797584896 bytes 2026-02-18T06:50:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1835105280 bytes 2026-02-18T06:50:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1845910528 bytes 2026-02-18T06:50:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1857435648 bytes 2026-02-18T06:50:59.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1877393408 bytes 2026-02-18T06:50:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1986614272 bytes 2026-02-18T06:50:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2000752640 bytes 2026-02-18T06:51:00.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2095542272 bytes 2026-02-18T06:51:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2115099648 bytes 2026-02-18T06:51:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2120860672 bytes 2026-02-18T06:51:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2139437056 bytes 2026-02-18T06:51:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2200071168 bytes 2026-02-18T06:51:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2202632192 bytes 2026-02-18T06:51:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2243418112 bytes 2026-02-18T06:51:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2261712896 bytes 2026-02-18T06:51:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2270521344 bytes 2026-02-18T06:51:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2352670720 bytes 2026-02-18T06:51:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2411055104 bytes 2026-02-18T06:51:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2421370880 bytes 2026-02-18T06:51:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2488836096 bytes 2026-02-18T06:51:00.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2518328320 bytes 2026-02-18T06:51:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2527654912 bytes 2026-02-18T06:51:00.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2548388864 bytes 2026-02-18T06:51:00.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2598201344 bytes 2026-02-18T06:51:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2636662784 bytes 2026-02-18T06:51:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2648128512 bytes 2026-02-18T06:51:00.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2662407168 bytes 2026-02-18T06:51:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2698997760 bytes 2026-02-18T06:51:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2705951744 bytes 2026-02-18T06:51:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2729902080 bytes 2026-02-18T06:51:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2809528320 bytes 2026-02-18T06:51:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2863039488 bytes 2026-02-18T06:51:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2882614272 bytes 2026-02-18T06:51:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2904814592 bytes 2026-02-18T06:51:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2960119808 bytes 2026-02-18T06:51:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3035029504 bytes 2026-02-18T06:51:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3075289088 bytes 2026-02-18T06:51:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3106202624 bytes 2026-02-18T06:51:01.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3187191808 bytes 2026-02-18T06:51:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3281119232 bytes 2026-02-18T06:51:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3341751296 bytes 2026-02-18T06:51:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3360630784 bytes 2026-02-18T06:51:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3367277568 bytes 2026-02-18T06:51:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3412035584 bytes 2026-02-18T06:51:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3418292224 bytes 2026-02-18T06:51:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3455939584 bytes 2026-02-18T06:51:02.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3456549888 bytes 2026-02-18T06:51:02.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3496648704 bytes 2026-02-18T06:51:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3523861504 bytes 2026-02-18T06:51:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3591219200 bytes 2026-02-18T06:51:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3607730176 bytes 2026-02-18T06:51:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3685479424 bytes 2026-02-18T06:51:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3695331328 bytes 2026-02-18T06:51:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3737708544 bytes 2026-02-18T06:51:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3778533376 bytes 2026-02-18T06:51:02.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3832104960 bytes 2026-02-18T06:51:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3880888320 bytes 2026-02-18T06:51:03.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3968449536 bytes 2026-02-18T06:51:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3986662400 bytes 2026-02-18T06:51:03.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4053724160 bytes 2026-02-18T06:51:03.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4076570624 bytes 2026-02-18T06:51:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4159070208 bytes 2026-02-18T06:51:03.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4195065856 bytes 2026-02-18T06:51:03.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4250783744 bytes 2026-02-18T06:51:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4254267392 bytes 2026-02-18T06:51:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4323927040 bytes 2026-02-18T06:51:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4362562560 bytes 2026-02-18T06:51:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4410439680 bytes 2026-02-18T06:51:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4447527936 bytes 2026-02-18T06:51:03.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4487042048 bytes 2026-02-18T06:51:03.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4508142592 bytes 2026-02-18T06:51:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4533450752 bytes 2026-02-18T06:51:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4594857984 bytes 2026-02-18T06:51:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4639500288 bytes 2026-02-18T06:51:04.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4688630784 bytes 2026-02-18T06:51:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4711714816 bytes 2026-02-18T06:51:04.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4777393152 bytes 2026-02-18T06:51:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4877438976 bytes 2026-02-18T06:51:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4926180352 bytes 2026-02-18T06:51:04.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4985402368 bytes 2026-02-18T06:51:04.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5029303296 bytes 2026-02-18T06:51:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5084251136 bytes 2026-02-18T06:51:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5110008832 bytes 2026-02-18T06:51:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5176524800 bytes 2026-02-18T06:51:04.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5284804608 bytes 2026-02-18T06:51:04.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5330298880 bytes 2026-02-18T06:51:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5378939904 bytes 2026-02-18T06:51:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5387279360 bytes 2026-02-18T06:51:05.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5426409472 bytes 2026-02-18T06:51:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5447019520 bytes 2026-02-18T06:51:05.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5511260160 bytes 2026-02-18T06:51:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5573443584 bytes 2026-02-18T06:51:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5638250496 bytes 2026-02-18T06:51:05.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5657766912 bytes 2026-02-18T06:51:05.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5703899136 bytes 2026-02-18T06:51:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5781588992 bytes 2026-02-18T06:51:05.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5859104768 bytes 2026-02-18T06:51:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5866297344 bytes 2026-02-18T06:51:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5933744128 bytes 2026-02-18T06:51:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5946791936 bytes 2026-02-18T06:51:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5969633280 bytes 2026-02-18T06:51:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6037405696 bytes 2026-02-18T06:51:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6069410816 bytes 2026-02-18T06:51:06.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6089269248 bytes 2026-02-18T06:51:06.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6147802112 bytes 2026-02-18T06:51:06.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6184314880 bytes 2026-02-18T06:51:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6232182784 bytes 2026-02-18T06:51:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6232897536 bytes 2026-02-18T06:51:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6234399744 bytes 2026-02-18T06:51:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6279845888 bytes 2026-02-18T06:51:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6296387584 bytes 2026-02-18T06:51:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6378850304 bytes 2026-02-18T06:51:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6420336640 bytes 2026-02-18T06:51:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6466204672 bytes 2026-02-18T06:51:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6522556416 bytes 2026-02-18T06:51:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6557870080 bytes 2026-02-18T06:51:06.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6598377472 bytes 2026-02-18T06:51:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6659777536 bytes 2026-02-18T06:51:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6672207872 bytes 2026-02-18T06:51:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6694409216 bytes 2026-02-18T06:51:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:51:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:51:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:51:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_32 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:51:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_32 2026-02-18T06:51:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:51:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:07.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2026-02-18T06:51:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:51:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2026-02-18T06:51:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_32': 2026-02-18T06:51:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:51:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14078533b0d0 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14078533b0d0 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:07 2026 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:07 2026 2026-02-18T06:51:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:07 2026 2026-02-18T06:51:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:51:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:51:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 4cc8e195-bdc7-4ab2-b53d-fb5ccf229629 2026-02-18T06:51:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:51:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:51:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:51:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:51:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:51:08.070 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:51:08.068+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f48b9a00 2026-02-18T06:51:08.070 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:51:08.068+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f4ac2d00 2026-02-18T06:51:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 24709120 bytes 2026-02-18T06:51:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 90856448 bytes 2026-02-18T06:51:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 119493632 bytes 2026-02-18T06:51:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 172612608 bytes 2026-02-18T06:51:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 197453824 bytes 2026-02-18T06:51:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 244301824 bytes 2026-02-18T06:51:08.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 299247616 bytes 2026-02-18T06:51:08.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 403470336 bytes 2026-02-18T06:51:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 436322304 bytes 2026-02-18T06:51:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 470277120 bytes 2026-02-18T06:51:08.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 494902272 bytes 2026-02-18T06:51:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 498837504 bytes 2026-02-18T06:51:08.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 530706432 bytes 2026-02-18T06:51:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 562021376 bytes 2026-02-18T06:51:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 628708352 bytes 2026-02-18T06:51:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 690831360 bytes 2026-02-18T06:51:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 696419328 bytes 2026-02-18T06:51:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 777610240 bytes 2026-02-18T06:51:09.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 830142464 bytes 2026-02-18T06:51:09.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 894792704 bytes 2026-02-18T06:51:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 939959296 bytes 2026-02-18T06:51:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 999195648 bytes 2026-02-18T06:51:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1006384128 bytes 2026-02-18T06:51:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1069314048 bytes 2026-02-18T06:51:09.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1123671040 bytes 2026-02-18T06:51:09.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1177090048 bytes 2026-02-18T06:51:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1213673472 bytes 2026-02-18T06:51:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1290874880 bytes 2026-02-18T06:51:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1311747072 bytes 2026-02-18T06:51:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1312327680 bytes 2026-02-18T06:51:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1364629504 bytes 2026-02-18T06:51:10.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1380691968 bytes 2026-02-18T06:51:10.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1417462784 bytes 2026-02-18T06:51:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1489756160 bytes 2026-02-18T06:51:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1554712576 bytes 2026-02-18T06:51:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1596446720 bytes 2026-02-18T06:51:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1651565568 bytes 2026-02-18T06:51:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1691530240 bytes 2026-02-18T06:51:10.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1744398336 bytes 2026-02-18T06:51:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1759965184 bytes 2026-02-18T06:51:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1792678912 bytes 2026-02-18T06:51:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1847029760 bytes 2026-02-18T06:51:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1916758016 bytes 2026-02-18T06:51:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1931614208 bytes 2026-02-18T06:51:10.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2013795328 bytes 2026-02-18T06:51:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2098976768 bytes 2026-02-18T06:51:11.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2120016896 bytes 2026-02-18T06:51:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2181972992 bytes 2026-02-18T06:51:11.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2221329408 bytes 2026-02-18T06:51:11.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2233058304 bytes 2026-02-18T06:51:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2267934720 bytes 2026-02-18T06:51:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2317440000 bytes 2026-02-18T06:51:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2370742272 bytes 2026-02-18T06:51:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2423925760 bytes 2026-02-18T06:51:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2424530944 bytes 2026-02-18T06:51:11.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2443414528 bytes 2026-02-18T06:51:11.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2444040192 bytes 2026-02-18T06:51:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2446045184 bytes 2026-02-18T06:51:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2511430656 bytes 2026-02-18T06:51:11.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2557895680 bytes 2026-02-18T06:51:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2587479040 bytes 2026-02-18T06:51:11.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2605603840 bytes 2026-02-18T06:51:11.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2626128896 bytes 2026-02-18T06:51:11.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2682749952 bytes 2026-02-18T06:51:11.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2690748416 bytes 2026-02-18T06:51:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2760499200 bytes 2026-02-18T06:51:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2789924864 bytes 2026-02-18T06:51:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2806479872 bytes 2026-02-18T06:51:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2834113536 bytes 2026-02-18T06:51:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2898994176 bytes 2026-02-18T06:51:12.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2912964608 bytes 2026-02-18T06:51:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2943188992 bytes 2026-02-18T06:51:12.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3079341056 bytes 2026-02-18T06:51:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3122430976 bytes 2026-02-18T06:51:12.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3132430336 bytes 2026-02-18T06:51:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3160976384 bytes 2026-02-18T06:51:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3236536320 bytes 2026-02-18T06:51:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3264408576 bytes 2026-02-18T06:51:12.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3339940864 bytes 2026-02-18T06:51:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3368341504 bytes 2026-02-18T06:51:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3380174848 bytes 2026-02-18T06:51:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3440866304 bytes 2026-02-18T06:51:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3467421696 bytes 2026-02-18T06:51:13.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3577950208 bytes 2026-02-18T06:51:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3597290496 bytes 2026-02-18T06:51:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3656284160 bytes 2026-02-18T06:51:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3711471616 bytes 2026-02-18T06:51:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3769880576 bytes 2026-02-18T06:51:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3775385600 bytes 2026-02-18T06:51:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3820939264 bytes 2026-02-18T06:51:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3850176512 bytes 2026-02-18T06:51:13.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3910142976 bytes 2026-02-18T06:51:13.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3983989760 bytes 2026-02-18T06:51:13.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4026863616 bytes 2026-02-18T06:51:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4074489856 bytes 2026-02-18T06:51:13.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4098489344 bytes 2026-02-18T06:51:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4160600064 bytes 2026-02-18T06:51:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4182067200 bytes 2026-02-18T06:51:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4270996480 bytes 2026-02-18T06:51:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4336352256 bytes 2026-02-18T06:51:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4373558272 bytes 2026-02-18T06:51:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4386935808 bytes 2026-02-18T06:51:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4387562496 bytes 2026-02-18T06:51:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4425121792 bytes 2026-02-18T06:51:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4461830144 bytes 2026-02-18T06:51:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4507933696 bytes 2026-02-18T06:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4536416256 bytes 2026-02-18T06:51:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4561069056 bytes 2026-02-18T06:51:14.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4574131200 bytes 2026-02-18T06:51:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4610842624 bytes 2026-02-18T06:51:14.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4671495168 bytes 2026-02-18T06:51:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4745253888 bytes 2026-02-18T06:51:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4776644608 bytes 2026-02-18T06:51:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4854879232 bytes 2026-02-18T06:51:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4869291008 bytes 2026-02-18T06:51:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4977784832 bytes 2026-02-18T06:51:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5016031232 bytes 2026-02-18T06:51:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5070342144 bytes 2026-02-18T06:51:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5130600448 bytes 2026-02-18T06:51:15.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5167367168 bytes 2026-02-18T06:51:15.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5216496640 bytes 2026-02-18T06:51:15.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5290083328 bytes 2026-02-18T06:51:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5319631872 bytes 2026-02-18T06:51:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5329209344 bytes 2026-02-18T06:51:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5356474368 bytes 2026-02-18T06:51:15.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5392660480 bytes 2026-02-18T06:51:15.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5484283904 bytes 2026-02-18T06:51:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5514747904 bytes 2026-02-18T06:51:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5537875968 bytes 2026-02-18T06:51:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5566080000 bytes 2026-02-18T06:51:16.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5586676736 bytes 2026-02-18T06:51:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5610448896 bytes 2026-02-18T06:51:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5666600960 bytes 2026-02-18T06:51:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5696152576 bytes 2026-02-18T06:51:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5785867264 bytes 2026-02-18T06:51:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5824792576 bytes 2026-02-18T06:51:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5843047424 bytes 2026-02-18T06:51:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5875784704 bytes 2026-02-18T06:51:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5959831552 bytes 2026-02-18T06:51:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6008625152 bytes 2026-02-18T06:51:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6072084480 bytes 2026-02-18T06:51:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6099547136 bytes 2026-02-18T06:51:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6143874048 bytes 2026-02-18T06:51:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6174725120 bytes 2026-02-18T06:51:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6221150208 bytes 2026-02-18T06:51:17.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6251860992 bytes 2026-02-18T06:51:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6287993856 bytes 2026-02-18T06:51:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6348370944 bytes 2026-02-18T06:51:17.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6370472960 bytes 2026-02-18T06:51:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6429518848 bytes 2026-02-18T06:51:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6457436160 bytes 2026-02-18T06:51:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6499945472 bytes 2026-02-18T06:51:17.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6537817088 bytes 2026-02-18T06:51:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6538420224 bytes 2026-02-18T06:51:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6599969792 bytes 2026-02-18T06:51:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6616116224 bytes 2026-02-18T06:51:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6699957248 bytes 2026-02-18T06:51:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6710530048 bytes 2026-02-18T06:51:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6769552384 bytes 2026-02-18T06:51:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6797475840 bytes 2026-02-18T06:51:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6863044608 bytes 2026-02-18T06:51:18.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6946271232 bytes 2026-02-18T06:51:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6979872768 bytes 2026-02-18T06:51:18.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7033319424 bytes 2026-02-18T06:51:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7051337728 bytes 2026-02-18T06:51:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7067788288 bytes 2026-02-18T06:51:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7084463104 bytes 2026-02-18T06:51:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7168509952 bytes 2026-02-18T06:51:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7174115328 bytes 2026-02-18T06:51:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7218616320 bytes 2026-02-18T06:51:18.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7291924480 bytes 2026-02-18T06:51:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7304025088 bytes 2026-02-18T06:51:18.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7310271488 bytes 2026-02-18T06:51:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7362422784 bytes 2026-02-18T06:51:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7376738304 bytes 2026-02-18T06:51:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7402338304 bytes 2026-02-18T06:51:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7414751232 bytes 2026-02-18T06:51:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7459009536 bytes 2026-02-18T06:51:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7479095296 bytes 2026-02-18T06:51:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7580775424 bytes 2026-02-18T06:51:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7601305600 bytes 2026-02-18T06:51:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7652926464 bytes 2026-02-18T06:51:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7707220992 bytes 2026-02-18T06:51:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7731946496 bytes 2026-02-18T06:51:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7754054656 bytes 2026-02-18T06:51:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7764766720 bytes 2026-02-18T06:51:19.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7836406784 bytes 2026-02-18T06:51:19.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7896130560 bytes 2026-02-18T06:51:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7946249216 bytes 2026-02-18T06:51:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7982633984 bytes 2026-02-18T06:51:19.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8008684544 bytes 2026-02-18T06:51:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8048956416 bytes 2026-02-18T06:51:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8094685184 bytes 2026-02-18T06:51:19.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8106678272 bytes 2026-02-18T06:51:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8165834752 bytes 2026-02-18T06:51:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8252864512 bytes 2026-02-18T06:51:20.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8279969792 bytes 2026-02-18T06:51:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8322040832 bytes 2026-02-18T06:51:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8351708160 bytes 2026-02-18T06:51:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8366460928 bytes 2026-02-18T06:51:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8387801088 bytes 2026-02-18T06:51:20.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8462797824 bytes 2026-02-18T06:51:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8543897600 bytes 2026-02-18T06:51:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8555821056 bytes 2026-02-18T06:51:20.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8635967488 bytes 2026-02-18T06:51:20.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8686256128 bytes 2026-02-18T06:51:20.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 8704776192 bytes 2026-02-18T06:51:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 8785324032 bytes 2026-02-18T06:51:20.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 8840738816 bytes 2026-02-18T06:51:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 8882914304 bytes 2026-02-18T06:51:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 8890949632 bytes 2026-02-18T06:51:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:51:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:51:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_33 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:51:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_33 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2026-02-18T06:51:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:51:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2026-02-18T06:51:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:51:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_33': 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14137203c1a0 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14137203c1a0 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:21 2026 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:21 2026 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:21 2026 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 72f9e03c-1614-48e6-b6e3-1f918706551e 2026-02-18T06:51:22.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:51:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:51:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-18T06:51:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:51:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:51:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=11 2026-02-18T06:51:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:51:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:51:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 65018880 bytes 2026-02-18T06:51:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 113254400 bytes 2026-02-18T06:51:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 216985600 bytes 2026-02-18T06:51:22.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 253372416 bytes 2026-02-18T06:51:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 263087104 bytes 2026-02-18T06:51:22.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 296301568 bytes 2026-02-18T06:51:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 378382336 bytes 2026-02-18T06:51:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 393393152 bytes 2026-02-18T06:51:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 439550976 bytes 2026-02-18T06:51:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 494389248 bytes 2026-02-18T06:51:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 534919168 bytes 2026-02-18T06:51:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 670397440 bytes 2026-02-18T06:51:23.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 758830080 bytes 2026-02-18T06:51:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 783289344 bytes 2026-02-18T06:51:23.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 784160768 bytes 2026-02-18T06:51:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 835956736 bytes 2026-02-18T06:51:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 863259648 bytes 2026-02-18T06:51:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 956049408 bytes 2026-02-18T06:51:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 1001233408 bytes 2026-02-18T06:51:23.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1059020800 bytes 2026-02-18T06:51:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1100661760 bytes 2026-02-18T06:51:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1203214336 bytes 2026-02-18T06:51:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1203830784 bytes 2026-02-18T06:51:23.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1252129792 bytes 2026-02-18T06:51:23.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1285526528 bytes 2026-02-18T06:51:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1286144000 bytes 2026-02-18T06:51:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1305674752 bytes 2026-02-18T06:51:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1355815936 bytes 2026-02-18T06:51:23.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1426037760 bytes 2026-02-18T06:51:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1478057984 bytes 2026-02-18T06:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1485921280 bytes 2026-02-18T06:51:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1506544640 bytes 2026-02-18T06:51:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1565955072 bytes 2026-02-18T06:51:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1637093376 bytes 2026-02-18T06:51:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1642674176 bytes 2026-02-18T06:51:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1681163264 bytes 2026-02-18T06:51:24.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1709223936 bytes 2026-02-18T06:51:24.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1733402624 bytes 2026-02-18T06:51:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1751003136 bytes 2026-02-18T06:51:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1800379392 bytes 2026-02-18T06:51:24.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1823420416 bytes 2026-02-18T06:51:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1858470912 bytes 2026-02-18T06:51:24.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1886410752 bytes 2026-02-18T06:51:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1903576064 bytes 2026-02-18T06:51:24.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1919015936 bytes 2026-02-18T06:51:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1964831744 bytes 2026-02-18T06:51:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1989234688 bytes 2026-02-18T06:51:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2003080192 bytes 2026-02-18T06:51:24.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2046237696 bytes 2026-02-18T06:51:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2080795648 bytes 2026-02-18T06:51:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2173074432 bytes 2026-02-18T06:51:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2183297024 bytes 2026-02-18T06:51:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2188505088 bytes 2026-02-18T06:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2262084608 bytes 2026-02-18T06:51:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2296136704 bytes 2026-02-18T06:51:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2364713984 bytes 2026-02-18T06:51:25.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2390501376 bytes 2026-02-18T06:51:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2434250752 bytes 2026-02-18T06:51:25.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2467628032 bytes 2026-02-18T06:51:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2581148672 bytes 2026-02-18T06:51:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2609088512 bytes 2026-02-18T06:51:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2670752768 bytes 2026-02-18T06:51:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2709336064 bytes 2026-02-18T06:51:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2745198592 bytes 2026-02-18T06:51:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2757561344 bytes 2026-02-18T06:51:26.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2789542912 bytes 2026-02-18T06:51:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2837359616 bytes 2026-02-18T06:51:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2899819520 bytes 2026-02-18T06:51:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2942519296 bytes 2026-02-18T06:51:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2962164736 bytes 2026-02-18T06:51:26.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2972867584 bytes 2026-02-18T06:51:26.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3001273344 bytes 2026-02-18T06:51:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3095753728 bytes 2026-02-18T06:51:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3096361984 bytes 2026-02-18T06:51:26.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3195831296 bytes 2026-02-18T06:51:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3269053440 bytes 2026-02-18T06:51:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3279291392 bytes 2026-02-18T06:51:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3328269312 bytes 2026-02-18T06:51:26.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3355744256 bytes 2026-02-18T06:51:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3407466496 bytes 2026-02-18T06:51:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3411412992 bytes 2026-02-18T06:51:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3430440960 bytes 2026-02-18T06:51:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3482787840 bytes 2026-02-18T06:51:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3592136704 bytes 2026-02-18T06:51:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3666605056 bytes 2026-02-18T06:51:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3699757056 bytes 2026-02-18T06:51:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3781886976 bytes 2026-02-18T06:51:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3833181184 bytes 2026-02-18T06:51:27.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3839757312 bytes 2026-02-18T06:51:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3931167744 bytes 2026-02-18T06:51:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3973640192 bytes 2026-02-18T06:51:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3978734592 bytes 2026-02-18T06:51:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4004156416 bytes 2026-02-18T06:51:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4062186496 bytes 2026-02-18T06:51:27.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4115119104 bytes 2026-02-18T06:51:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4132713472 bytes 2026-02-18T06:51:27.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4165179392 bytes 2026-02-18T06:51:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4205485056 bytes 2026-02-18T06:51:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4235786240 bytes 2026-02-18T06:51:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4256967680 bytes 2026-02-18T06:51:28.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4319550464 bytes 2026-02-18T06:51:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4344467456 bytes 2026-02-18T06:51:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4388773888 bytes 2026-02-18T06:51:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4429092864 bytes 2026-02-18T06:51:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4503798784 bytes 2026-02-18T06:51:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4560735232 bytes 2026-02-18T06:51:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4561354752 bytes 2026-02-18T06:51:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4597370880 bytes 2026-02-18T06:51:28.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4613477376 bytes 2026-02-18T06:51:28.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4629325824 bytes 2026-02-18T06:51:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4651088896 bytes 2026-02-18T06:51:28.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4691749888 bytes 2026-02-18T06:51:28.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4771439616 bytes 2026-02-18T06:51:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4829825024 bytes 2026-02-18T06:51:28.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4925798400 bytes 2026-02-18T06:51:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5009596416 bytes 2026-02-18T06:51:29.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5076682752 bytes 2026-02-18T06:51:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5085228032 bytes 2026-02-18T06:51:29.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5138032640 bytes 2026-02-18T06:51:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5138646016 bytes 2026-02-18T06:51:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5175581696 bytes 2026-02-18T06:51:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5260193792 bytes 2026-02-18T06:51:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5287982080 bytes 2026-02-18T06:51:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5308500992 bytes 2026-02-18T06:51:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5328550912 bytes 2026-02-18T06:51:29.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5394467840 bytes 2026-02-18T06:51:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5415481344 bytes 2026-02-18T06:51:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5483342848 bytes 2026-02-18T06:51:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5523901440 bytes 2026-02-18T06:51:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5576972288 bytes 2026-02-18T06:51:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5639566336 bytes 2026-02-18T06:51:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5676902400 bytes 2026-02-18T06:51:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5763055616 bytes 2026-02-18T06:51:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5825641472 bytes 2026-02-18T06:51:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5861187584 bytes 2026-02-18T06:51:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5903940608 bytes 2026-02-18T06:51:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5987491840 bytes 2026-02-18T06:51:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6049068032 bytes 2026-02-18T06:51:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6069161984 bytes 2026-02-18T06:51:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6101671936 bytes 2026-02-18T06:51:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6102287360 bytes 2026-02-18T06:51:30.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6151851008 bytes 2026-02-18T06:51:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6231685120 bytes 2026-02-18T06:51:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6300039168 bytes 2026-02-18T06:51:30.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6370151424 bytes 2026-02-18T06:51:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6398863360 bytes 2026-02-18T06:51:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6425920512 bytes 2026-02-18T06:51:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6439345152 bytes 2026-02-18T06:51:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6476670976 bytes 2026-02-18T06:51:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6521469952 bytes 2026-02-18T06:51:31.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6597638144 bytes 2026-02-18T06:51:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6617942016 bytes 2026-02-18T06:51:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6681222144 bytes 2026-02-18T06:51:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6727067648 bytes 2026-02-18T06:51:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6755129344 bytes 2026-02-18T06:51:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6784367616 bytes 2026-02-18T06:51:31.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6788664320 bytes 2026-02-18T06:51:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6838983680 bytes 2026-02-18T06:51:31.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6878620672 bytes 2026-02-18T06:51:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6915955712 bytes 2026-02-18T06:51:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7012179968 bytes 2026-02-18T06:51:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7084358656 bytes 2026-02-18T06:51:31.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7098635264 bytes 2026-02-18T06:51:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7110968320 bytes 2026-02-18T06:51:32.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7191321600 bytes 2026-02-18T06:51:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7231428608 bytes 2026-02-18T06:51:32.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7290713088 bytes 2026-02-18T06:51:32.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7310013440 bytes 2026-02-18T06:51:32.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7376078848 bytes 2026-02-18T06:51:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7428039680 bytes 2026-02-18T06:51:32.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7483849728 bytes 2026-02-18T06:51:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7511238656 bytes 2026-02-18T06:51:32.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7578221568 bytes 2026-02-18T06:51:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7631900672 bytes 2026-02-18T06:51:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7707020288 bytes 2026-02-18T06:51:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7729583104 bytes 2026-02-18T06:51:32.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7804862464 bytes 2026-02-18T06:51:32.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7830786048 bytes 2026-02-18T06:51:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7930401792 bytes 2026-02-18T06:51:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7998756864 bytes 2026-02-18T06:51:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:51:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:51:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:51:33.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_34 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:51:33.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:33.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_34 2026-02-18T06:51:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:51:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:51:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2026-02-18T06:51:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:51:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_34': 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 141fe35045a6 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.141fe35045a6 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:51:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:33 2026 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:33 2026 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:33 2026 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1c347546-ec8a-4a65-84e2-e2bd5d979b24 2026-02-18T06:51:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:51:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:51:34.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:51:34.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:51:34.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:51:34.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 78045184 bytes 2026-02-18T06:51:34.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 147898368 bytes 2026-02-18T06:51:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 150687744 bytes 2026-02-18T06:51:34.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 218689536 bytes 2026-02-18T06:51:34.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 238352384 bytes 2026-02-18T06:51:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 257426432 bytes 2026-02-18T06:51:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 320999424 bytes 2026-02-18T06:51:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 371584000 bytes 2026-02-18T06:51:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 421364736 bytes 2026-02-18T06:51:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 446601216 bytes 2026-02-18T06:51:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 496776192 bytes 2026-02-18T06:51:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 502112256 bytes 2026-02-18T06:51:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 520109056 bytes 2026-02-18T06:51:34.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 555248640 bytes 2026-02-18T06:51:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 585579520 bytes 2026-02-18T06:51:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 589002752 bytes 2026-02-18T06:51:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 658911232 bytes 2026-02-18T06:51:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 747283456 bytes 2026-02-18T06:51:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 778999808 bytes 2026-02-18T06:51:35.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 801459200 bytes 2026-02-18T06:51:35.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 837826560 bytes 2026-02-18T06:51:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 872935424 bytes 2026-02-18T06:51:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 924663808 bytes 2026-02-18T06:51:35.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 966282240 bytes 2026-02-18T06:51:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 994720768 bytes 2026-02-18T06:51:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1047769088 bytes 2026-02-18T06:51:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1066232832 bytes 2026-02-18T06:51:35.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1136851968 bytes 2026-02-18T06:51:35.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1196068864 bytes 2026-02-18T06:51:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1208846336 bytes 2026-02-18T06:51:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1255021568 bytes 2026-02-18T06:51:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1260105728 bytes 2026-02-18T06:51:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1329359872 bytes 2026-02-18T06:51:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1359759360 bytes 2026-02-18T06:51:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1413219328 bytes 2026-02-18T06:51:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1450381312 bytes 2026-02-18T06:51:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1475276800 bytes 2026-02-18T06:51:36.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1530336256 bytes 2026-02-18T06:51:36.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1596001280 bytes 2026-02-18T06:51:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1612823552 bytes 2026-02-18T06:51:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1625619456 bytes 2026-02-18T06:51:36.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1626225664 bytes 2026-02-18T06:51:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1716045824 bytes 2026-02-18T06:51:36.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1754125312 bytes 2026-02-18T06:51:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1801422848 bytes 2026-02-18T06:51:36.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1823097856 bytes 2026-02-18T06:51:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1855311872 bytes 2026-02-18T06:51:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1931847680 bytes 2026-02-18T06:51:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1956846592 bytes 2026-02-18T06:51:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1977226240 bytes 2026-02-18T06:51:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1987411968 bytes 2026-02-18T06:51:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2071716864 bytes 2026-02-18T06:51:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2119560192 bytes 2026-02-18T06:51:37.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2140794880 bytes 2026-02-18T06:51:37.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2239091712 bytes 2026-02-18T06:51:37.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2242083840 bytes 2026-02-18T06:51:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2264954880 bytes 2026-02-18T06:51:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2369828864 bytes 2026-02-18T06:51:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2370423808 bytes 2026-02-18T06:51:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2398972928 bytes 2026-02-18T06:51:37.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2412222464 bytes 2026-02-18T06:51:37.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2486123520 bytes 2026-02-18T06:51:37.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2553079808 bytes 2026-02-18T06:51:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2617374720 bytes 2026-02-18T06:51:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2660043776 bytes 2026-02-18T06:51:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2691299328 bytes 2026-02-18T06:51:38.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2714487808 bytes 2026-02-18T06:51:38.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2763772928 bytes 2026-02-18T06:51:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2797158400 bytes 2026-02-18T06:51:38.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2848080896 bytes 2026-02-18T06:51:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2883261440 bytes 2026-02-18T06:51:38.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2905617408 bytes 2026-02-18T06:51:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2930368512 bytes 2026-02-18T06:51:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2955584512 bytes 2026-02-18T06:51:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3041261568 bytes 2026-02-18T06:51:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3061237760 bytes 2026-02-18T06:51:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3079092224 bytes 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:51:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_35 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-18T06:51:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_35 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:39.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:51:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:39.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2026-02-18T06:51:40.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:51:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_35': 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 142bc58c18b9 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.142bc58c18b9 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:51:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:39 2026 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:39 2026 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:39 2026 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 26bf0afd-33b6-476b-b97e-600b51d08d3b 2026-02-18T06:51:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:51:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:51:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=10 2026-02-18T06:51:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:51:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:51:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 90091520 bytes 2026-02-18T06:51:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 135776256 bytes 2026-02-18T06:51:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 174426112 bytes 2026-02-18T06:51:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 269877248 bytes 2026-02-18T06:51:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 334155776 bytes 2026-02-18T06:51:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 413768704 bytes 2026-02-18T06:51:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 444378112 bytes 2026-02-18T06:51:40.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 446507008 bytes 2026-02-18T06:51:40.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 515410944 bytes 2026-02-18T06:51:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 596185088 bytes 2026-02-18T06:51:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 620182528 bytes 2026-02-18T06:51:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 695993344 bytes 2026-02-18T06:51:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 779448320 bytes 2026-02-18T06:51:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 782575616 bytes 2026-02-18T06:51:41.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 846704640 bytes 2026-02-18T06:51:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 872603648 bytes 2026-02-18T06:51:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 885852160 bytes 2026-02-18T06:51:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 929449984 bytes 2026-02-18T06:51:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 983308288 bytes 2026-02-18T06:51:41.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1021585408 bytes 2026-02-18T06:51:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1064670208 bytes 2026-02-18T06:51:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1118099456 bytes 2026-02-18T06:51:41.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1155778560 bytes 2026-02-18T06:51:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1156392960 bytes 2026-02-18T06:51:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1173732352 bytes 2026-02-18T06:51:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1174315008 bytes 2026-02-18T06:51:41.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1184688128 bytes 2026-02-18T06:51:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1216025600 bytes 2026-02-18T06:51:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1271156736 bytes 2026-02-18T06:51:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1324486656 bytes 2026-02-18T06:51:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1343214592 bytes 2026-02-18T06:51:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1370308608 bytes 2026-02-18T06:51:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1384048640 bytes 2026-02-18T06:51:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1430585344 bytes 2026-02-18T06:51:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1528196096 bytes 2026-02-18T06:51:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1560841216 bytes 2026-02-18T06:51:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1608390656 bytes 2026-02-18T06:51:42.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1688824832 bytes 2026-02-18T06:51:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1700202496 bytes 2026-02-18T06:51:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1747759104 bytes 2026-02-18T06:51:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1810366464 bytes 2026-02-18T06:51:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1830468608 bytes 2026-02-18T06:51:42.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1905775616 bytes 2026-02-18T06:51:42.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1944610816 bytes 2026-02-18T06:51:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1945242624 bytes 2026-02-18T06:51:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2021657600 bytes 2026-02-18T06:51:42.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2066426880 bytes 2026-02-18T06:51:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2095094784 bytes 2026-02-18T06:51:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2135480320 bytes 2026-02-18T06:51:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2179096576 bytes 2026-02-18T06:51:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2186540032 bytes 2026-02-18T06:51:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2242478080 bytes 2026-02-18T06:51:43.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2275730432 bytes 2026-02-18T06:51:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2297555968 bytes 2026-02-18T06:51:43.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2348389376 bytes 2026-02-18T06:51:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2408105984 bytes 2026-02-18T06:51:43.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2442805248 bytes 2026-02-18T06:51:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2512749568 bytes 2026-02-18T06:51:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2522943488 bytes 2026-02-18T06:51:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2573509632 bytes 2026-02-18T06:51:43.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2613453824 bytes 2026-02-18T06:51:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2654844928 bytes 2026-02-18T06:51:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2659799040 bytes 2026-02-18T06:51:43.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2749889536 bytes 2026-02-18T06:51:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2809999360 bytes 2026-02-18T06:51:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2876762112 bytes 2026-02-18T06:51:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2922934272 bytes 2026-02-18T06:51:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2957238272 bytes 2026-02-18T06:51:44.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3021065216 bytes 2026-02-18T06:51:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3068581888 bytes 2026-02-18T06:51:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3137428480 bytes 2026-02-18T06:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3185796096 bytes 2026-02-18T06:51:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3211032576 bytes 2026-02-18T06:51:44.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3238359040 bytes 2026-02-18T06:51:44.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3274312704 bytes 2026-02-18T06:51:44.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3304867840 bytes 2026-02-18T06:51:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3347942400 bytes 2026-02-18T06:51:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3348552704 bytes 2026-02-18T06:51:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3397314560 bytes 2026-02-18T06:51:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3410722816 bytes 2026-02-18T06:51:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3462724608 bytes 2026-02-18T06:51:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3499125760 bytes 2026-02-18T06:51:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3563623424 bytes 2026-02-18T06:51:45.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3565099008 bytes 2026-02-18T06:51:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3625544704 bytes 2026-02-18T06:51:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3661028352 bytes 2026-02-18T06:51:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3712528384 bytes 2026-02-18T06:51:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3760415744 bytes 2026-02-18T06:51:45.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3801282560 bytes 2026-02-18T06:51:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3856634880 bytes 2026-02-18T06:51:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3926692864 bytes 2026-02-18T06:51:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3947484160 bytes 2026-02-18T06:51:45.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3980619776 bytes 2026-02-18T06:51:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3981228032 bytes 2026-02-18T06:51:45.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3985577984 bytes 2026-02-18T06:51:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4048516096 bytes 2026-02-18T06:51:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4087553024 bytes 2026-02-18T06:51:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4116840448 bytes 2026-02-18T06:51:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4177921024 bytes 2026-02-18T06:51:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4232879104 bytes 2026-02-18T06:51:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4284900352 bytes 2026-02-18T06:51:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4338648064 bytes 2026-02-18T06:51:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4377629696 bytes 2026-02-18T06:51:46.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4393945088 bytes 2026-02-18T06:51:46.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4394568704 bytes 2026-02-18T06:51:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4444809216 bytes 2026-02-18T06:51:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4512706560 bytes 2026-02-18T06:51:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4577152000 bytes 2026-02-18T06:51:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4671649792 bytes 2026-02-18T06:51:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4695826432 bytes 2026-02-18T06:51:46.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4743849984 bytes 2026-02-18T06:51:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4843556864 bytes 2026-02-18T06:51:47.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4865247232 bytes 2026-02-18T06:51:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4904931328 bytes 2026-02-18T06:51:47.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4941491200 bytes 2026-02-18T06:51:47.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4980857856 bytes 2026-02-18T06:51:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4981472256 bytes 2026-02-18T06:51:47.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5011031040 bytes 2026-02-18T06:51:47.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5064358912 bytes 2026-02-18T06:51:47.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5115074560 bytes 2026-02-18T06:51:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5149224960 bytes 2026-02-18T06:51:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5154823168 bytes 2026-02-18T06:51:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5231283200 bytes 2026-02-18T06:51:47.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5231895552 bytes 2026-02-18T06:51:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5327671296 bytes 2026-02-18T06:51:47.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5363135488 bytes 2026-02-18T06:51:47.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5370153984 bytes 2026-02-18T06:51:47.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5388698624 bytes 2026-02-18T06:51:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5443352576 bytes 2026-02-18T06:51:47.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5490083840 bytes 2026-02-18T06:51:48.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5534835712 bytes 2026-02-18T06:51:48.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5603940352 bytes 2026-02-18T06:51:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5610407936 bytes 2026-02-18T06:51:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5658532864 bytes 2026-02-18T06:51:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5754273792 bytes 2026-02-18T06:51:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5784264704 bytes 2026-02-18T06:51:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5805012992 bytes 2026-02-18T06:51:48.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5826965504 bytes 2026-02-18T06:51:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5836988416 bytes 2026-02-18T06:51:48.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5884348416 bytes 2026-02-18T06:51:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5936279552 bytes 2026-02-18T06:51:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5972505600 bytes 2026-02-18T06:51:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5992235008 bytes 2026-02-18T06:51:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6020141056 bytes 2026-02-18T06:51:48.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6054575104 bytes 2026-02-18T06:51:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6140536832 bytes 2026-02-18T06:51:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6236187648 bytes 2026-02-18T06:51:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6315740160 bytes 2026-02-18T06:51:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6351504384 bytes 2026-02-18T06:51:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6375289856 bytes 2026-02-18T06:51:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6414658560 bytes 2026-02-18T06:51:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6432347136 bytes 2026-02-18T06:51:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6432947200 bytes 2026-02-18T06:51:49.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6466286592 bytes 2026-02-18T06:51:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6538688512 bytes 2026-02-18T06:51:49.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6574393344 bytes 2026-02-18T06:51:49.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6649658368 bytes 2026-02-18T06:51:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6725413888 bytes 2026-02-18T06:51:49.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6768702464 bytes 2026-02-18T06:51:49.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6802456576 bytes 2026-02-18T06:51:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6811196416 bytes 2026-02-18T06:51:49.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6843995136 bytes 2026-02-18T06:51:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6894023680 bytes 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_36 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:51:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_36 2026-02-18T06:51:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:51:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:51:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:51:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2026-02-18T06:51:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:51:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_36': 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 143760fdb8b3 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.143760fdb8b3 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:51:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:50 2026 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:50 2026 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:50 2026 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 0c2518ac-f371-4e3c-855f-dcdd605f3efc 2026-02-18T06:51:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:51:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:51:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=10 2026-02-18T06:51:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:51:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:51:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 42787840 bytes 2026-02-18T06:51:51.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 143532032 bytes 2026-02-18T06:51:51.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 166279168 bytes 2026-02-18T06:51:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 228244480 bytes 2026-02-18T06:51:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 248738816 bytes 2026-02-18T06:51:51.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 300656640 bytes 2026-02-18T06:51:51.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 350435328 bytes 2026-02-18T06:51:51.582 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9491121013340118) > weight (0.25) 2026-02-18T06:51:51.582 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4846079955497087) > weight (0.25) 2026-02-18T06:51:51.583 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7604365779979347) > weight (0.25) 2026-02-18T06:51:51.583 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6834745420998655) > weight (0.25) 2026-02-18T06:51:51.583 INFO:tasks.rbd_mirror_thrash:waiting for 105.687463674736 secs before thrashing 2026-02-18T06:51:51.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 413675520 bytes 2026-02-18T06:51:51.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 414965760 bytes 2026-02-18T06:51:51.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 428214272 bytes 2026-02-18T06:51:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 496828416 bytes 2026-02-18T06:51:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 535797760 bytes 2026-02-18T06:51:51.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 584470528 bytes 2026-02-18T06:51:51.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 585098240 bytes 2026-02-18T06:51:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 604729344 bytes 2026-02-18T06:51:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 625717248 bytes 2026-02-18T06:51:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 651193344 bytes 2026-02-18T06:51:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 667202560 bytes 2026-02-18T06:51:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 703917056 bytes 2026-02-18T06:51:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 743752704 bytes 2026-02-18T06:51:52.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 801888256 bytes 2026-02-18T06:51:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 865867776 bytes 2026-02-18T06:51:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 890072064 bytes 2026-02-18T06:51:52.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 935053312 bytes 2026-02-18T06:51:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 995632128 bytes 2026-02-18T06:51:52.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1047537664 bytes 2026-02-18T06:51:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1098878976 bytes 2026-02-18T06:51:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1167732736 bytes 2026-02-18T06:51:52.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1195807744 bytes 2026-02-18T06:51:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1224600576 bytes 2026-02-18T06:51:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1231732736 bytes 2026-02-18T06:51:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1361858560 bytes 2026-02-18T06:51:53.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1445847040 bytes 2026-02-18T06:51:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1486887936 bytes 2026-02-18T06:51:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1526859776 bytes 2026-02-18T06:51:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1589388288 bytes 2026-02-18T06:51:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1632122880 bytes 2026-02-18T06:51:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1667054592 bytes 2026-02-18T06:51:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1667683328 bytes 2026-02-18T06:51:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1724756992 bytes 2026-02-18T06:51:53.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1761756160 bytes 2026-02-18T06:51:53.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1819360256 bytes 2026-02-18T06:51:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1865967616 bytes 2026-02-18T06:51:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1917620224 bytes 2026-02-18T06:51:53.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1987621888 bytes 2026-02-18T06:51:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2032824320 bytes 2026-02-18T06:51:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2136424448 bytes 2026-02-18T06:51:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2155895808 bytes 2026-02-18T06:51:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2234086400 bytes 2026-02-18T06:51:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2314916864 bytes 2026-02-18T06:51:54.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2354785280 bytes 2026-02-18T06:51:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2434509824 bytes 2026-02-18T06:51:54.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2485215232 bytes 2026-02-18T06:51:54.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2527687680 bytes 2026-02-18T06:51:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2575930368 bytes 2026-02-18T06:51:54.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2626379776 bytes 2026-02-18T06:51:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2660489216 bytes 2026-02-18T06:51:54.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2710246400 bytes 2026-02-18T06:51:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2727643136 bytes 2026-02-18T06:51:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2787183616 bytes 2026-02-18T06:51:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2833902592 bytes 2026-02-18T06:51:55.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2871030784 bytes 2026-02-18T06:51:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2936770560 bytes 2026-02-18T06:51:55.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2937414656 bytes 2026-02-18T06:51:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2961157120 bytes 2026-02-18T06:51:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2999250944 bytes 2026-02-18T06:51:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3051820032 bytes 2026-02-18T06:51:55.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3089748992 bytes 2026-02-18T06:51:55.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3130417152 bytes 2026-02-18T06:51:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3154987008 bytes 2026-02-18T06:51:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3201336320 bytes 2026-02-18T06:51:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3257752576 bytes 2026-02-18T06:51:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3288607744 bytes 2026-02-18T06:51:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3374627840 bytes 2026-02-18T06:51:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3450888192 bytes 2026-02-18T06:51:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3463872512 bytes 2026-02-18T06:51:55.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3477253120 bytes 2026-02-18T06:51:56.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3553823744 bytes 2026-02-18T06:51:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3655587840 bytes 2026-02-18T06:51:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3678547968 bytes 2026-02-18T06:51:56.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3703513088 bytes 2026-02-18T06:51:56.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3713582080 bytes 2026-02-18T06:51:56.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3746260992 bytes 2026-02-18T06:51:56.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3816523776 bytes 2026-02-18T06:51:56.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3862352896 bytes 2026-02-18T06:51:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3910432768 bytes 2026-02-18T06:51:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3929887744 bytes 2026-02-18T06:51:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3983633408 bytes 2026-02-18T06:51:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3984259072 bytes 2026-02-18T06:51:56.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4046759936 bytes 2026-02-18T06:51:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4071946240 bytes 2026-02-18T06:51:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4092956672 bytes 2026-02-18T06:51:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4143011840 bytes 2026-02-18T06:51:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4181778432 bytes 2026-02-18T06:51:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4191688704 bytes 2026-02-18T06:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4229505024 bytes 2026-02-18T06:51:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4247806976 bytes 2026-02-18T06:51:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4311401472 bytes 2026-02-18T06:51:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4317830144 bytes 2026-02-18T06:51:57.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4347754496 bytes 2026-02-18T06:51:57.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4394434560 bytes 2026-02-18T06:51:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4463648768 bytes 2026-02-18T06:51:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4555624448 bytes 2026-02-18T06:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4576082944 bytes 2026-02-18T06:51:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4633499648 bytes 2026-02-18T06:51:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4667566080 bytes 2026-02-18T06:51:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4706340864 bytes 2026-02-18T06:51:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4716813312 bytes 2026-02-18T06:51:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4738299904 bytes 2026-02-18T06:51:57.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4776716288 bytes 2026-02-18T06:51:58.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4814016512 bytes 2026-02-18T06:51:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4890083328 bytes 2026-02-18T06:51:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4938900480 bytes 2026-02-18T06:51:58.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4951794688 bytes 2026-02-18T06:51:58.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4975287296 bytes 2026-02-18T06:51:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5045762048 bytes 2026-02-18T06:51:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5125988352 bytes 2026-02-18T06:51:58.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5193583616 bytes 2026-02-18T06:51:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5220144128 bytes 2026-02-18T06:51:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5275640832 bytes 2026-02-18T06:51:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5286823936 bytes 2026-02-18T06:51:58.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5316988928 bytes 2026-02-18T06:51:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5337009152 bytes 2026-02-18T06:51:58.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5385101312 bytes 2026-02-18T06:51:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5421361152 bytes 2026-02-18T06:51:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5484464128 bytes 2026-02-18T06:51:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5523519488 bytes 2026-02-18T06:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5555352576 bytes 2026-02-18T06:51:59.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5614366720 bytes 2026-02-18T06:51:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5638208512 bytes 2026-02-18T06:51:59.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5671907328 bytes 2026-02-18T06:51:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5701355520 bytes 2026-02-18T06:51:59.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5728419840 bytes 2026-02-18T06:51:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5769275392 bytes 2026-02-18T06:51:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5869855744 bytes 2026-02-18T06:51:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5917604864 bytes 2026-02-18T06:51:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5993980928 bytes 2026-02-18T06:51:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6069366784 bytes 2026-02-18T06:51:59.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6115041280 bytes 2026-02-18T06:51:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6133121024 bytes 2026-02-18T06:51:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6173413376 bytes 2026-02-18T06:52:00.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6217925632 bytes 2026-02-18T06:52:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6320091136 bytes 2026-02-18T06:52:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6371893248 bytes 2026-02-18T06:52:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6396491776 bytes 2026-02-18T06:52:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6426831872 bytes 2026-02-18T06:52:00.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6482211840 bytes 2026-02-18T06:52:00.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6543570944 bytes 2026-02-18T06:52:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6578430976 bytes 2026-02-18T06:52:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6606689280 bytes 2026-02-18T06:52:00.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6683543552 bytes 2026-02-18T06:52:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6715924480 bytes 2026-02-18T06:52:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6764825600 bytes 2026-02-18T06:52:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6833566720 bytes 2026-02-18T06:52:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6859745280 bytes 2026-02-18T06:52:01.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6924299264 bytes 2026-02-18T06:52:01.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6925871104 bytes 2026-02-18T06:52:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_37 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:52:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:01.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:01.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_37 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:52:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2026-02-18T06:52:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_37': 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1443d5f1c031 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1443d5f1c031 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:02.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:01 2026 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:01 2026 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:01 2026 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 2d943d33-932e-4e88-9180-1e3322e2eaf7 2026-02-18T06:52:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-18T06:52:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:52:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:52:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 17966080 bytes 2026-02-18T06:52:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 18577408 bytes 2026-02-18T06:52:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 56776704 bytes 2026-02-18T06:52:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 116904960 bytes 2026-02-18T06:52:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 136788992 bytes 2026-02-18T06:52:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 193813504 bytes 2026-02-18T06:52:02.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 218600448 bytes 2026-02-18T06:52:02.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 278244352 bytes 2026-02-18T06:52:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 329604096 bytes 2026-02-18T06:52:02.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 397428736 bytes 2026-02-18T06:52:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 421494784 bytes 2026-02-18T06:52:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 422135808 bytes 2026-02-18T06:52:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 422734848 bytes 2026-02-18T06:52:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 453931008 bytes 2026-02-18T06:52:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 462917632 bytes 2026-02-18T06:52:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 480904192 bytes 2026-02-18T06:52:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 503284736 bytes 2026-02-18T06:52:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 557197312 bytes 2026-02-18T06:52:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 578443264 bytes 2026-02-18T06:52:03.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 626820096 bytes 2026-02-18T06:52:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 657784832 bytes 2026-02-18T06:52:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 697054208 bytes 2026-02-18T06:52:03.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 713741312 bytes 2026-02-18T06:52:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 751577088 bytes 2026-02-18T06:52:03.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 820643840 bytes 2026-02-18T06:52:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 823924736 bytes 2026-02-18T06:52:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 824536064 bytes 2026-02-18T06:52:03.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 856388608 bytes 2026-02-18T06:52:03.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 889303040 bytes 2026-02-18T06:52:03.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 936425472 bytes 2026-02-18T06:52:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 948868096 bytes 2026-02-18T06:52:03.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 991147008 bytes 2026-02-18T06:52:03.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1088277504 bytes 2026-02-18T06:52:03.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1156845568 bytes 2026-02-18T06:52:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1187434496 bytes 2026-02-18T06:52:04.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1242850304 bytes 2026-02-18T06:52:04.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1282998272 bytes 2026-02-18T06:52:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1338561536 bytes 2026-02-18T06:52:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1346772992 bytes 2026-02-18T06:52:04.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1355327488 bytes 2026-02-18T06:52:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1380585472 bytes 2026-02-18T06:52:04.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1399229440 bytes 2026-02-18T06:52:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1419988992 bytes 2026-02-18T06:52:04.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1492494336 bytes 2026-02-18T06:52:04.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1570254848 bytes 2026-02-18T06:52:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1575419904 bytes 2026-02-18T06:52:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1626012672 bytes 2026-02-18T06:52:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1633586176 bytes 2026-02-18T06:52:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1656248320 bytes 2026-02-18T06:52:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1681310720 bytes 2026-02-18T06:52:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1749577728 bytes 2026-02-18T06:52:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1814100992 bytes 2026-02-18T06:52:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1901790208 bytes 2026-02-18T06:52:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1966583808 bytes 2026-02-18T06:52:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2023322624 bytes 2026-02-18T06:52:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2091561984 bytes 2026-02-18T06:52:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2110636032 bytes 2026-02-18T06:52:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2189576192 bytes 2026-02-18T06:52:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2245703680 bytes 2026-02-18T06:52:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2316982272 bytes 2026-02-18T06:52:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2346101760 bytes 2026-02-18T06:52:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2365079552 bytes 2026-02-18T06:52:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2406193152 bytes 2026-02-18T06:52:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2457988096 bytes 2026-02-18T06:52:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2507458560 bytes 2026-02-18T06:52:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2529138688 bytes 2026-02-18T06:52:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2571381760 bytes 2026-02-18T06:52:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2612679680 bytes 2026-02-18T06:52:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2696740864 bytes 2026-02-18T06:52:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2725972992 bytes 2026-02-18T06:52:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2790119424 bytes 2026-02-18T06:52:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2876224512 bytes 2026-02-18T06:52:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2925402112 bytes 2026-02-18T06:52:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2937243648 bytes 2026-02-18T06:52:06.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2995757056 bytes 2026-02-18T06:52:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3010370560 bytes 2026-02-18T06:52:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3059764224 bytes 2026-02-18T06:52:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3071360000 bytes 2026-02-18T06:52:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3126281216 bytes 2026-02-18T06:52:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3143919616 bytes 2026-02-18T06:52:07.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3201060864 bytes 2026-02-18T06:52:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_38 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_38 2026-02-18T06:52:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2026-02-18T06:52:08.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_38': 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 144fad1feca0 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.144fad1feca0 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:07 2026 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:07 2026 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:07 2026 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c3f5441a-66c5-4aac-8a5c-285f1d0190dd 2026-02-18T06:52:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-18T06:52:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T06:52:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T06:52:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 27500544 bytes 2026-02-18T06:52:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 88428544 bytes 2026-02-18T06:52:08.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 137467904 bytes 2026-02-18T06:52:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 178830336 bytes 2026-02-18T06:52:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 208402432 bytes 2026-02-18T06:52:08.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 266489856 bytes 2026-02-18T06:52:08.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 311636992 bytes 2026-02-18T06:52:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 359684096 bytes 2026-02-18T06:52:08.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 382656512 bytes 2026-02-18T06:52:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 404707328 bytes 2026-02-18T06:52:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 444587008 bytes 2026-02-18T06:52:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 473294848 bytes 2026-02-18T06:52:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 510247936 bytes 2026-02-18T06:52:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 531680256 bytes 2026-02-18T06:52:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 577842176 bytes 2026-02-18T06:52:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 639868928 bytes 2026-02-18T06:52:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 671728640 bytes 2026-02-18T06:52:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 710545408 bytes 2026-02-18T06:52:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 750437376 bytes 2026-02-18T06:52:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 845492224 bytes 2026-02-18T06:52:09.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 873152512 bytes 2026-02-18T06:52:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 925023232 bytes 2026-02-18T06:52:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 966534144 bytes 2026-02-18T06:52:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1018133504 bytes 2026-02-18T06:52:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1070756864 bytes 2026-02-18T06:52:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1148595200 bytes 2026-02-18T06:52:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1203804160 bytes 2026-02-18T06:52:09.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1290805248 bytes 2026-02-18T06:52:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1319290880 bytes 2026-02-18T06:52:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1387250688 bytes 2026-02-18T06:52:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1399254016 bytes 2026-02-18T06:52:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1399871488 bytes 2026-02-18T06:52:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1496951808 bytes 2026-02-18T06:52:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1536856064 bytes 2026-02-18T06:52:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1576509440 bytes 2026-02-18T06:52:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1595515904 bytes 2026-02-18T06:52:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1638079488 bytes 2026-02-18T06:52:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1689535488 bytes 2026-02-18T06:52:10.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1717874688 bytes 2026-02-18T06:52:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1726861312 bytes 2026-02-18T06:52:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1818397696 bytes 2026-02-18T06:52:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1826512896 bytes 2026-02-18T06:52:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1838891008 bytes 2026-02-18T06:52:10.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1926141952 bytes 2026-02-18T06:52:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1934484480 bytes 2026-02-18T06:52:10.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1935082496 bytes 2026-02-18T06:52:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1939048448 bytes 2026-02-18T06:52:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1984045056 bytes 2026-02-18T06:52:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2007550976 bytes 2026-02-18T06:52:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2025149440 bytes 2026-02-18T06:52:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2071075840 bytes 2026-02-18T06:52:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2100875264 bytes 2026-02-18T06:52:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2142110720 bytes 2026-02-18T06:52:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2152255488 bytes 2026-02-18T06:52:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2193040384 bytes 2026-02-18T06:52:11.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2203674624 bytes 2026-02-18T06:52:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2236154880 bytes 2026-02-18T06:52:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2336418816 bytes 2026-02-18T06:52:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2379104256 bytes 2026-02-18T06:52:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2403012608 bytes 2026-02-18T06:52:11.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2459653120 bytes 2026-02-18T06:52:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2522270720 bytes 2026-02-18T06:52:12.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2539083776 bytes 2026-02-18T06:52:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2539666432 bytes 2026-02-18T06:52:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2569442304 bytes 2026-02-18T06:52:12.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2596443136 bytes 2026-02-18T06:52:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2647332864 bytes 2026-02-18T06:52:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2691667968 bytes 2026-02-18T06:52:12.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2761559040 bytes 2026-02-18T06:52:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2806556672 bytes 2026-02-18T06:52:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2859308032 bytes 2026-02-18T06:52:12.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2893151232 bytes 2026-02-18T06:52:12.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2997825536 bytes 2026-02-18T06:52:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3027577856 bytes 2026-02-18T06:52:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3036563456 bytes 2026-02-18T06:52:12.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3047523328 bytes 2026-02-18T06:52:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3110628352 bytes 2026-02-18T06:52:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3200471040 bytes 2026-02-18T06:52:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3223716864 bytes 2026-02-18T06:52:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3293395968 bytes 2026-02-18T06:52:13.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3332233216 bytes 2026-02-18T06:52:13.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3405731840 bytes 2026-02-18T06:52:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3444905984 bytes 2026-02-18T06:52:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3499061248 bytes 2026-02-18T06:52:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3518094336 bytes 2026-02-18T06:52:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3541381120 bytes 2026-02-18T06:52:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3546096640 bytes 2026-02-18T06:52:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3618606080 bytes 2026-02-18T06:52:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3666286592 bytes 2026-02-18T06:52:13.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3689848832 bytes 2026-02-18T06:52:13.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3741734912 bytes 2026-02-18T06:52:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3783089152 bytes 2026-02-18T06:52:13.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3844390912 bytes 2026-02-18T06:52:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3854356480 bytes 2026-02-18T06:52:13.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3885793280 bytes 2026-02-18T06:52:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3929523200 bytes 2026-02-18T06:52:14.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3945951232 bytes 2026-02-18T06:52:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3979097088 bytes 2026-02-18T06:52:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4011018240 bytes 2026-02-18T06:52:14.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4053671936 bytes 2026-02-18T06:52:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4169841664 bytes 2026-02-18T06:52:14.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4262807552 bytes 2026-02-18T06:52:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4294867968 bytes 2026-02-18T06:52:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4382409728 bytes 2026-02-18T06:52:14.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4401842176 bytes 2026-02-18T06:52:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4446074880 bytes 2026-02-18T06:52:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4475964416 bytes 2026-02-18T06:52:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4574570496 bytes 2026-02-18T06:52:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4652435456 bytes 2026-02-18T06:52:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4692225024 bytes 2026-02-18T06:52:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4728102912 bytes 2026-02-18T06:52:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4802766848 bytes 2026-02-18T06:52:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4803380224 bytes 2026-02-18T06:52:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4828009472 bytes 2026-02-18T06:52:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4843421696 bytes 2026-02-18T06:52:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4919967744 bytes 2026-02-18T06:52:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4959371264 bytes 2026-02-18T06:52:15.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4986001408 bytes 2026-02-18T06:52:15.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5000538112 bytes 2026-02-18T06:52:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5024624640 bytes 2026-02-18T06:52:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5073900544 bytes 2026-02-18T06:52:15.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5142861824 bytes 2026-02-18T06:52:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5165711360 bytes 2026-02-18T06:52:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5173573632 bytes 2026-02-18T06:52:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5190914048 bytes 2026-02-18T06:52:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5234110464 bytes 2026-02-18T06:52:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5270747136 bytes 2026-02-18T06:52:16.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5351151616 bytes 2026-02-18T06:52:16.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5384052736 bytes 2026-02-18T06:52:16.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5441863680 bytes 2026-02-18T06:52:16.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5460569088 bytes 2026-02-18T06:52:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5530997760 bytes 2026-02-18T06:52:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5578193920 bytes 2026-02-18T06:52:16.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5652104192 bytes 2026-02-18T06:52:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5697121280 bytes 2026-02-18T06:52:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5830288384 bytes 2026-02-18T06:52:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5830902784 bytes 2026-02-18T06:52:16.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5878255616 bytes 2026-02-18T06:52:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5917455360 bytes 2026-02-18T06:52:16.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5979039744 bytes 2026-02-18T06:52:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6016876544 bytes 2026-02-18T06:52:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6113905664 bytes 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_39 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_39 2026-02-18T06:52:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2026-02-18T06:52:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T06:52:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2026-02-18T06:52:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_39': 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 145bf2323b10 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.145bf2323b10 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:17 2026 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:17 2026 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:17 2026 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1e647b33-04b4-4af9-be40-448754ebc6b9 2026-02-18T06:52:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-18T06:52:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T06:52:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T06:52:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:17.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 22500352 bytes 2026-02-18T06:52:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 56087552 bytes 2026-02-18T06:52:17.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 91294720 bytes 2026-02-18T06:52:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 112433152 bytes 2026-02-18T06:52:17.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 141572096 bytes 2026-02-18T06:52:17.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 154396672 bytes 2026-02-18T06:52:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 216900608 bytes 2026-02-18T06:52:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 236921856 bytes 2026-02-18T06:52:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 256190464 bytes 2026-02-18T06:52:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 298847232 bytes 2026-02-18T06:52:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 342222848 bytes 2026-02-18T06:52:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 454788096 bytes 2026-02-18T06:52:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 497319936 bytes 2026-02-18T06:52:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 524296192 bytes 2026-02-18T06:52:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 542498816 bytes 2026-02-18T06:52:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 643095552 bytes 2026-02-18T06:52:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 683867136 bytes 2026-02-18T06:52:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 737568768 bytes 2026-02-18T06:52:18.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 784642048 bytes 2026-02-18T06:52:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 821504000 bytes 2026-02-18T06:52:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 858990592 bytes 2026-02-18T06:52:18.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 930436096 bytes 2026-02-18T06:52:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1004382208 bytes 2026-02-18T06:52:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1089040384 bytes 2026-02-18T06:52:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1114764288 bytes 2026-02-18T06:52:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1223158784 bytes 2026-02-18T06:52:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1255898112 bytes 2026-02-18T06:52:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1326050304 bytes 2026-02-18T06:52:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1328974848 bytes 2026-02-18T06:52:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1438563328 bytes 2026-02-18T06:52:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1540958208 bytes 2026-02-18T06:52:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1618125824 bytes 2026-02-18T06:52:19.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1646058496 bytes 2026-02-18T06:52:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1695188992 bytes 2026-02-18T06:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1737284608 bytes 2026-02-18T06:52:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1771886592 bytes 2026-02-18T06:52:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1801360384 bytes 2026-02-18T06:52:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1851261952 bytes 2026-02-18T06:52:19.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1879544832 bytes 2026-02-18T06:52:19.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1914552320 bytes 2026-02-18T06:52:19.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1925614592 bytes 2026-02-18T06:52:19.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1948600320 bytes 2026-02-18T06:52:19.999 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:52:19.996+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbf8baadd0 2026-02-18T06:52:19.999 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:52:19.996+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbf8babc70 2026-02-18T06:52:19.999 INFO:tasks.ceph.cluster2.osd.0.trial051.stderr:2026-02-18T06:52:19.996+0000 7ff3f3d03640 -1 reset not still connected to 0x55bbfa913930 2026-02-18T06:52:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1985403904 bytes 2026-02-18T06:52:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2040517632 bytes 2026-02-18T06:52:20.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2089484288 bytes 2026-02-18T06:52:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2129694720 bytes 2026-02-18T06:52:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2208548864 bytes 2026-02-18T06:52:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2235924480 bytes 2026-02-18T06:52:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2299054080 bytes 2026-02-18T06:52:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2330777600 bytes 2026-02-18T06:52:20.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2399467520 bytes 2026-02-18T06:52:20.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2435084288 bytes 2026-02-18T06:52:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2462899200 bytes 2026-02-18T06:52:20.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2510912512 bytes 2026-02-18T06:52:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2555332608 bytes 2026-02-18T06:52:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2580234240 bytes 2026-02-18T06:52:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2637754368 bytes 2026-02-18T06:52:20.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2708092928 bytes 2026-02-18T06:52:21.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2750058496 bytes 2026-02-18T06:52:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2750672896 bytes 2026-02-18T06:52:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2778388480 bytes 2026-02-18T06:52:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2860709888 bytes 2026-02-18T06:52:21.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2892811264 bytes 2026-02-18T06:52:21.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2991783936 bytes 2026-02-18T06:52:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3014767616 bytes 2026-02-18T06:52:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3061096448 bytes 2026-02-18T06:52:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3091706880 bytes 2026-02-18T06:52:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3149092864 bytes 2026-02-18T06:52:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3165557760 bytes 2026-02-18T06:52:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3172277248 bytes 2026-02-18T06:52:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3234142208 bytes 2026-02-18T06:52:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3268828160 bytes 2026-02-18T06:52:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3327109120 bytes 2026-02-18T06:52:21.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3332524032 bytes 2026-02-18T06:52:21.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3367531520 bytes 2026-02-18T06:52:22.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3416774656 bytes 2026-02-18T06:52:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3473750016 bytes 2026-02-18T06:52:22.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3527237632 bytes 2026-02-18T06:52:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_40 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_40 2026-02-18T06:52:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2026-02-18T06:52:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2026-02-18T06:52:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_40': 2026-02-18T06:52:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1467bdc79b02 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1467bdc79b02 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:22 2026 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:22 2026 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:22 2026 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d79ef20b-93e5-4c9a-898c-892bd3f13eb7 2026-02-18T06:52:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-18T06:52:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T06:52:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=14 2026-02-18T06:52:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 61427712 bytes 2026-02-18T06:52:23.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 117986304 bytes 2026-02-18T06:52:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 133352448 bytes 2026-02-18T06:52:23.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 192486400 bytes 2026-02-18T06:52:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 229392384 bytes 2026-02-18T06:52:23.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 234702848 bytes 2026-02-18T06:52:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 287015936 bytes 2026-02-18T06:52:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 333979648 bytes 2026-02-18T06:52:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 371266560 bytes 2026-02-18T06:52:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 428469248 bytes 2026-02-18T06:52:23.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 521265152 bytes 2026-02-18T06:52:23.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 548505600 bytes 2026-02-18T06:52:23.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 586344448 bytes 2026-02-18T06:52:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 636785664 bytes 2026-02-18T06:52:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 637373440 bytes 2026-02-18T06:52:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 660494336 bytes 2026-02-18T06:52:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 715099136 bytes 2026-02-18T06:52:24.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 715728896 bytes 2026-02-18T06:52:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 740165632 bytes 2026-02-18T06:52:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 792678400 bytes 2026-02-18T06:52:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 851328000 bytes 2026-02-18T06:52:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 862810112 bytes 2026-02-18T06:52:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 888200192 bytes 2026-02-18T06:52:24.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 953493504 bytes 2026-02-18T06:52:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1014045696 bytes 2026-02-18T06:52:24.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1061753856 bytes 2026-02-18T06:52:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1097970688 bytes 2026-02-18T06:52:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1106657280 bytes 2026-02-18T06:52:24.776 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3259058124684858) > weight (0.25) 2026-02-18T06:52:24.776 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.819441621790541) > weight (0.25) 2026-02-18T06:52:24.776 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.45649631801326196) > weight (0.25) 2026-02-18T06:52:24.776 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5207285216811841) > weight (0.25) 2026-02-18T06:52:24.776 INFO:tasks.rbd_mirror_thrash:waiting for 103.2414246769404 secs before thrashing 2026-02-18T06:52:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1170139136 bytes 2026-02-18T06:52:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1183741952 bytes 2026-02-18T06:52:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1231477760 bytes 2026-02-18T06:52:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1239124992 bytes 2026-02-18T06:52:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1282765824 bytes 2026-02-18T06:52:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1327002624 bytes 2026-02-18T06:52:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1376375808 bytes 2026-02-18T06:52:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1444458496 bytes 2026-02-18T06:52:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1494204416 bytes 2026-02-18T06:52:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1559006208 bytes 2026-02-18T06:52:25.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1626483712 bytes 2026-02-18T06:52:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1660567552 bytes 2026-02-18T06:52:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1679874048 bytes 2026-02-18T06:52:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1690900480 bytes 2026-02-18T06:52:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1721819136 bytes 2026-02-18T06:52:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1801728000 bytes 2026-02-18T06:52:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1817596928 bytes 2026-02-18T06:52:26.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1876692992 bytes 2026-02-18T06:52:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1935684608 bytes 2026-02-18T06:52:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2025781248 bytes 2026-02-18T06:52:26.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2041709568 bytes 2026-02-18T06:52:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2060741632 bytes 2026-02-18T06:52:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2144019456 bytes 2026-02-18T06:52:26.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2177508352 bytes 2026-02-18T06:52:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2225203200 bytes 2026-02-18T06:52:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2267218944 bytes 2026-02-18T06:52:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2301305856 bytes 2026-02-18T06:52:26.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2394771456 bytes 2026-02-18T06:52:26.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2426265600 bytes 2026-02-18T06:52:26.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2489393152 bytes 2026-02-18T06:52:26.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2539627520 bytes 2026-02-18T06:52:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2633759744 bytes 2026-02-18T06:52:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2680680448 bytes 2026-02-18T06:52:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2724390912 bytes 2026-02-18T06:52:27.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2769668096 bytes 2026-02-18T06:52:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2809377792 bytes 2026-02-18T06:52:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2843813888 bytes 2026-02-18T06:52:27.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2884852736 bytes 2026-02-18T06:52:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2906869760 bytes 2026-02-18T06:52:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3038454784 bytes 2026-02-18T06:52:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3090070528 bytes 2026-02-18T06:52:27.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3148593152 bytes 2026-02-18T06:52:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3166743552 bytes 2026-02-18T06:52:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3242194944 bytes 2026-02-18T06:52:27.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3281910784 bytes 2026-02-18T06:52:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3348431872 bytes 2026-02-18T06:52:28.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3381737472 bytes 2026-02-18T06:52:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3407836160 bytes 2026-02-18T06:52:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3429287936 bytes 2026-02-18T06:52:28.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3479235584 bytes 2026-02-18T06:52:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3512515584 bytes 2026-02-18T06:52:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3521928192 bytes 2026-02-18T06:52:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3572100096 bytes 2026-02-18T06:52:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3624749056 bytes 2026-02-18T06:52:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3734782976 bytes 2026-02-18T06:52:28.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3751350272 bytes 2026-02-18T06:52:28.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3784236032 bytes 2026-02-18T06:52:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3794011136 bytes 2026-02-18T06:52:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3824454656 bytes 2026-02-18T06:52:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3894174720 bytes 2026-02-18T06:52:28.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3989253120 bytes 2026-02-18T06:52:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4040807424 bytes 2026-02-18T06:52:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4056939520 bytes 2026-02-18T06:52:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4112787456 bytes 2026-02-18T06:52:29.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4141460480 bytes 2026-02-18T06:52:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4167003136 bytes 2026-02-18T06:52:29.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4179958784 bytes 2026-02-18T06:52:29.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4209584128 bytes 2026-02-18T06:52:29.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4283014144 bytes 2026-02-18T06:52:29.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4320154624 bytes 2026-02-18T06:52:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4361240576 bytes 2026-02-18T06:52:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4400393216 bytes 2026-02-18T06:52:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4523695104 bytes 2026-02-18T06:52:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4568000512 bytes 2026-02-18T06:52:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4629680128 bytes 2026-02-18T06:52:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4672545792 bytes 2026-02-18T06:52:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4683153408 bytes 2026-02-18T06:52:29.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4696514560 bytes 2026-02-18T06:52:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4753435648 bytes 2026-02-18T06:52:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4771524608 bytes 2026-02-18T06:52:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4787185664 bytes 2026-02-18T06:52:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4822361088 bytes 2026-02-18T06:52:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4831891456 bytes 2026-02-18T06:52:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4890133504 bytes 2026-02-18T06:52:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4913060864 bytes 2026-02-18T06:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4938396672 bytes 2026-02-18T06:52:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5006301184 bytes 2026-02-18T06:52:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5061627904 bytes 2026-02-18T06:52:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5151420416 bytes 2026-02-18T06:52:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5182292992 bytes 2026-02-18T06:52:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5280011264 bytes 2026-02-18T06:52:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5368258560 bytes 2026-02-18T06:52:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5407405056 bytes 2026-02-18T06:52:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5540448256 bytes 2026-02-18T06:52:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5602242560 bytes 2026-02-18T06:52:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5665116160 bytes 2026-02-18T06:52:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5728202752 bytes 2026-02-18T06:52:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5749584896 bytes 2026-02-18T06:52:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5917639680 bytes 2026-02-18T06:52:31.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5930683392 bytes 2026-02-18T06:52:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5972207616 bytes 2026-02-18T06:52:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5989473280 bytes 2026-02-18T06:52:31.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 6012704768 bytes 2026-02-18T06:52:31.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 6057200640 bytes 2026-02-18T06:52:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6135146496 bytes 2026-02-18T06:52:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6184920064 bytes 2026-02-18T06:52:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6189419520 bytes 2026-02-18T06:52:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6205220864 bytes 2026-02-18T06:52:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6240481280 bytes 2026-02-18T06:52:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6286830592 bytes 2026-02-18T06:52:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6331237376 bytes 2026-02-18T06:52:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6376739840 bytes 2026-02-18T06:52:32.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6384843776 bytes 2026-02-18T06:52:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6428252160 bytes 2026-02-18T06:52:32.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6471925760 bytes 2026-02-18T06:52:32.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6545690624 bytes 2026-02-18T06:52:32.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6603103232 bytes 2026-02-18T06:52:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6640507904 bytes 2026-02-18T06:52:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6703721472 bytes 2026-02-18T06:52:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6739469312 bytes 2026-02-18T06:52:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6802408448 bytes 2026-02-18T06:52:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6840691712 bytes 2026-02-18T06:52:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6841325568 bytes 2026-02-18T06:52:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6850750464 bytes 2026-02-18T06:52:33.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6878884864 bytes 2026-02-18T06:52:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6897653760 bytes 2026-02-18T06:52:33.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6932091904 bytes 2026-02-18T06:52:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6962408448 bytes 2026-02-18T06:52:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7086841856 bytes 2026-02-18T06:52:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7142716416 bytes 2026-02-18T06:52:33.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7159303168 bytes 2026-02-18T06:52:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7173603328 bytes 2026-02-18T06:52:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7241078784 bytes 2026-02-18T06:52:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7309238272 bytes 2026-02-18T06:52:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7359305728 bytes 2026-02-18T06:52:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7381749760 bytes 2026-02-18T06:52:33.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7406966784 bytes 2026-02-18T06:52:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7425101824 bytes 2026-02-18T06:52:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7445831680 bytes 2026-02-18T06:52:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7446411264 bytes 2026-02-18T06:52:33.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7514334208 bytes 2026-02-18T06:52:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7530617856 bytes 2026-02-18T06:52:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7593657344 bytes 2026-02-18T06:52:34.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7702383616 bytes 2026-02-18T06:52:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7767374848 bytes 2026-02-18T06:52:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7785580544 bytes 2026-02-18T06:52:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7870789632 bytes 2026-02-18T06:52:34.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7919687680 bytes 2026-02-18T06:52:34.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7953249280 bytes 2026-02-18T06:52:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7976668160 bytes 2026-02-18T06:52:34.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 8023418880 bytes 2026-02-18T06:52:34.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 8060896256 bytes 2026-02-18T06:52:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 8125162496 bytes 2026-02-18T06:52:34.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 8187506688 bytes 2026-02-18T06:52:34.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 8193472512 bytes 2026-02-18T06:52:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 8227918848 bytes 2026-02-18T06:52:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8263218176 bytes 2026-02-18T06:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8323093504 bytes 2026-02-18T06:52:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8372732928 bytes 2026-02-18T06:52:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8405667840 bytes 2026-02-18T06:52:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8418897920 bytes 2026-02-18T06:52:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8428896256 bytes 2026-02-18T06:52:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8500204544 bytes 2026-02-18T06:52:35.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8555756544 bytes 2026-02-18T06:52:35.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8597778432 bytes 2026-02-18T06:52:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8652175360 bytes 2026-02-18T06:52:35.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8742445056 bytes 2026-02-18T06:52:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8777905152 bytes 2026-02-18T06:52:35.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8836378624 bytes 2026-02-18T06:52:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8838190080 bytes 2026-02-18T06:52:35.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8855307264 bytes 2026-02-18T06:52:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8879490048 bytes 2026-02-18T06:52:36.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8922485760 bytes 2026-02-18T06:52:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8951276544 bytes 2026-02-18T06:52:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8985405440 bytes 2026-02-18T06:52:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 9016992768 bytes 2026-02-18T06:52:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 9098501120 bytes 2026-02-18T06:52:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 9129182208 bytes 2026-02-18T06:52:36.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20700 IOs, 9230887936 bytes 2026-02-18T06:52:36.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20800 IOs, 9250982912 bytes 2026-02-18T06:52:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20900 IOs, 9272595456 bytes 2026-02-18T06:52:36.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21000 IOs, 9343364096 bytes 2026-02-18T06:52:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21100 IOs, 9343972352 bytes 2026-02-18T06:52:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21200 IOs, 9356698624 bytes 2026-02-18T06:52:36.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21300 IOs, 9448929280 bytes 2026-02-18T06:52:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21400 IOs, 9518134272 bytes 2026-02-18T06:52:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21500 IOs, 9560999936 bytes 2026-02-18T06:52:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:21600 IOs, 9636143104 bytes 2026-02-18T06:52:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:37.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_41 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_41 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T06:52:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2026-02-18T06:52:37.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:37.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_41': 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14734b3ab14b 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14734b3ab14b 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:37 2026 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:37 2026 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:37 2026 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d085af16-259e-4393-9922-61c84825c9b6 2026-02-18T06:52:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T06:52:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:52:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 42464256 bytes 2026-02-18T06:52:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 112191488 bytes 2026-02-18T06:52:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 190339072 bytes 2026-02-18T06:52:37.486 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:52:37.483+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f7c25790 2026-02-18T06:52:37.486 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:52:37.483+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fb180dd0 2026-02-18T06:52:37.486 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:52:37.484+0000 7f8d89fae640 -1 reset not still connected to 0x55c815a720d0 2026-02-18T06:52:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 222035968 bytes 2026-02-18T06:52:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 269716480 bytes 2026-02-18T06:52:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 272495616 bytes 2026-02-18T06:52:37.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 343870464 bytes 2026-02-18T06:52:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 394081280 bytes 2026-02-18T06:52:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 419000320 bytes 2026-02-18T06:52:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 467574784 bytes 2026-02-18T06:52:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 478401536 bytes 2026-02-18T06:52:38.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 533361664 bytes 2026-02-18T06:52:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 600583168 bytes 2026-02-18T06:52:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 627914752 bytes 2026-02-18T06:52:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 683098112 bytes 2026-02-18T06:52:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 683672576 bytes 2026-02-18T06:52:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 721471488 bytes 2026-02-18T06:52:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 821998592 bytes 2026-02-18T06:52:38.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 835339264 bytes 2026-02-18T06:52:38.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 857084928 bytes 2026-02-18T06:52:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 898653184 bytes 2026-02-18T06:52:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 936584192 bytes 2026-02-18T06:52:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 940726272 bytes 2026-02-18T06:52:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 958949376 bytes 2026-02-18T06:52:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 980241408 bytes 2026-02-18T06:52:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1060963328 bytes 2026-02-18T06:52:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1070135296 bytes 2026-02-18T06:52:38.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1110318080 bytes 2026-02-18T06:52:38.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1130927104 bytes 2026-02-18T06:52:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1159916544 bytes 2026-02-18T06:52:39.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1231377408 bytes 2026-02-18T06:52:39.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1294175232 bytes 2026-02-18T06:52:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1298401280 bytes 2026-02-18T06:52:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1303438336 bytes 2026-02-18T06:52:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1420430336 bytes 2026-02-18T06:52:39.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1446081536 bytes 2026-02-18T06:52:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1482441728 bytes 2026-02-18T06:52:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1545093120 bytes 2026-02-18T06:52:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1591284736 bytes 2026-02-18T06:52:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1644017664 bytes 2026-02-18T06:52:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1693678592 bytes 2026-02-18T06:52:39.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1741246464 bytes 2026-02-18T06:52:39.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1803494400 bytes 2026-02-18T06:52:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1836049408 bytes 2026-02-18T06:52:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1929928704 bytes 2026-02-18T06:52:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1985709056 bytes 2026-02-18T06:52:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2027456512 bytes 2026-02-18T06:52:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2076895232 bytes 2026-02-18T06:52:40.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2122124288 bytes 2026-02-18T06:52:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2164284416 bytes 2026-02-18T06:52:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2170624000 bytes 2026-02-18T06:52:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2190861312 bytes 2026-02-18T06:52:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2238113792 bytes 2026-02-18T06:52:40.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2311853056 bytes 2026-02-18T06:52:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2312462336 bytes 2026-02-18T06:52:40.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2313074688 bytes 2026-02-18T06:52:40.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2361885696 bytes 2026-02-18T06:52:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2405336064 bytes 2026-02-18T06:52:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2430917632 bytes 2026-02-18T06:52:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2440387584 bytes 2026-02-18T06:52:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2497405952 bytes 2026-02-18T06:52:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2525826048 bytes 2026-02-18T06:52:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2590267392 bytes 2026-02-18T06:52:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2672786432 bytes 2026-02-18T06:52:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2713406464 bytes 2026-02-18T06:52:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2749425664 bytes 2026-02-18T06:52:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2808668160 bytes 2026-02-18T06:52:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2856854528 bytes 2026-02-18T06:52:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2920181760 bytes 2026-02-18T06:52:41.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2984176640 bytes 2026-02-18T06:52:41.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3038874624 bytes 2026-02-18T06:52:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3094538240 bytes 2026-02-18T06:52:42.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3157872640 bytes 2026-02-18T06:52:42.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3182940160 bytes 2026-02-18T06:52:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3207037952 bytes 2026-02-18T06:52:42.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3252997120 bytes 2026-02-18T06:52:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3277410304 bytes 2026-02-18T06:52:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3293379584 bytes 2026-02-18T06:52:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3308165120 bytes 2026-02-18T06:52:42.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3369705472 bytes 2026-02-18T06:52:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3428123648 bytes 2026-02-18T06:52:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3433226240 bytes 2026-02-18T06:52:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3448174592 bytes 2026-02-18T06:52:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3523713024 bytes 2026-02-18T06:52:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3547696128 bytes 2026-02-18T06:52:42.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3604216832 bytes 2026-02-18T06:52:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3679909888 bytes 2026-02-18T06:52:42.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3714724864 bytes 2026-02-18T06:52:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3730930688 bytes 2026-02-18T06:52:43.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3770055680 bytes 2026-02-18T06:52:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3805428736 bytes 2026-02-18T06:52:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3840950272 bytes 2026-02-18T06:52:43.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3877925888 bytes 2026-02-18T06:52:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3898270720 bytes 2026-02-18T06:52:43.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3919848448 bytes 2026-02-18T06:52:43.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3920457728 bytes 2026-02-18T06:52:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4007600128 bytes 2026-02-18T06:52:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4100220928 bytes 2026-02-18T06:52:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4194219008 bytes 2026-02-18T06:52:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4233921536 bytes 2026-02-18T06:52:43.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4252770304 bytes 2026-02-18T06:52:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4310718464 bytes 2026-02-18T06:52:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4354810880 bytes 2026-02-18T06:52:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4383947776 bytes 2026-02-18T06:52:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4399044608 bytes 2026-02-18T06:52:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4426156032 bytes 2026-02-18T06:52:44.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4491144192 bytes 2026-02-18T06:52:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4541080576 bytes 2026-02-18T06:52:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4543845376 bytes 2026-02-18T06:52:44.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4578010112 bytes 2026-02-18T06:52:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4594202624 bytes 2026-02-18T06:52:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4645624832 bytes 2026-02-18T06:52:44.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4695533568 bytes 2026-02-18T06:52:44.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4721215488 bytes 2026-02-18T06:52:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4745641984 bytes 2026-02-18T06:52:44.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4772664320 bytes 2026-02-18T06:52:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4822164480 bytes 2026-02-18T06:52:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4865875968 bytes 2026-02-18T06:52:44.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4906155008 bytes 2026-02-18T06:52:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4953009152 bytes 2026-02-18T06:52:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5006801920 bytes 2026-02-18T06:52:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5007415296 bytes 2026-02-18T06:52:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5055837184 bytes 2026-02-18T06:52:45.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5076967424 bytes 2026-02-18T06:52:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5077583872 bytes 2026-02-18T06:52:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5098798080 bytes 2026-02-18T06:52:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5140399104 bytes 2026-02-18T06:52:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5176559616 bytes 2026-02-18T06:52:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5256759296 bytes 2026-02-18T06:52:45.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5282433024 bytes 2026-02-18T06:52:45.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5299525632 bytes 2026-02-18T06:52:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5364436992 bytes 2026-02-18T06:52:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5398228992 bytes 2026-02-18T06:52:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5431189504 bytes 2026-02-18T06:52:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5505050624 bytes 2026-02-18T06:52:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5537003520 bytes 2026-02-18T06:52:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5547314176 bytes 2026-02-18T06:52:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5553142784 bytes 2026-02-18T06:52:45.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5583628288 bytes 2026-02-18T06:52:46.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5618334720 bytes 2026-02-18T06:52:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5664983040 bytes 2026-02-18T06:52:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5694623744 bytes 2026-02-18T06:52:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5748444160 bytes 2026-02-18T06:52:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5787440128 bytes 2026-02-18T06:52:46.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5856884736 bytes 2026-02-18T06:52:46.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5879792640 bytes 2026-02-18T06:52:46.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5920459776 bytes 2026-02-18T06:52:46.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5986795520 bytes 2026-02-18T06:52:46.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6039190528 bytes 2026-02-18T06:52:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6082020352 bytes 2026-02-18T06:52:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6122631168 bytes 2026-02-18T06:52:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6141424640 bytes 2026-02-18T06:52:46.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6157819904 bytes 2026-02-18T06:52:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6168563712 bytes 2026-02-18T06:52:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6217038848 bytes 2026-02-18T06:52:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6246555648 bytes 2026-02-18T06:52:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6267310080 bytes 2026-02-18T06:52:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6286244864 bytes 2026-02-18T06:52:47.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6313430016 bytes 2026-02-18T06:52:47.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6333701120 bytes 2026-02-18T06:52:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6356744192 bytes 2026-02-18T06:52:47.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6387008512 bytes 2026-02-18T06:52:47.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6398370816 bytes 2026-02-18T06:52:47.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6484150272 bytes 2026-02-18T06:52:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6540343296 bytes 2026-02-18T06:52:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6563771392 bytes 2026-02-18T06:52:47.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6599788544 bytes 2026-02-18T06:52:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6630566912 bytes 2026-02-18T06:52:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6724842496 bytes 2026-02-18T06:52:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6772054016 bytes 2026-02-18T06:52:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6808049664 bytes 2026-02-18T06:52:48.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6885774336 bytes 2026-02-18T06:52:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6959949824 bytes 2026-02-18T06:52:48.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6998993920 bytes 2026-02-18T06:52:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7087100928 bytes 2026-02-18T06:52:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7134919680 bytes 2026-02-18T06:52:48.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7186726912 bytes 2026-02-18T06:52:48.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7237613568 bytes 2026-02-18T06:52:48.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7250796544 bytes 2026-02-18T06:52:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7300381696 bytes 2026-02-18T06:52:48.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7336155136 bytes 2026-02-18T06:52:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7381722112 bytes 2026-02-18T06:52:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7426558976 bytes 2026-02-18T06:52:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7467969536 bytes 2026-02-18T06:52:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7497891840 bytes 2026-02-18T06:52:48.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7498516480 bytes 2026-02-18T06:52:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7571331072 bytes 2026-02-18T06:52:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7594667008 bytes 2026-02-18T06:52:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7645620224 bytes 2026-02-18T06:52:49.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7726987264 bytes 2026-02-18T06:52:49.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7828373504 bytes 2026-02-18T06:52:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7882474496 bytes 2026-02-18T06:52:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7947040768 bytes 2026-02-18T06:52:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8026856448 bytes 2026-02-18T06:52:49.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8128223232 bytes 2026-02-18T06:52:49.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8209951744 bytes 2026-02-18T06:52:49.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8245134336 bytes 2026-02-18T06:52:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8274125824 bytes 2026-02-18T06:52:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8292390912 bytes 2026-02-18T06:52:50.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8298910720 bytes 2026-02-18T06:52:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8324103168 bytes 2026-02-18T06:52:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8389182464 bytes 2026-02-18T06:52:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:52:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:52:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:52:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:50.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_42 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:52:50.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:52:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_42 2026-02-18T06:52:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:52:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2026-02-18T06:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:52:51.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2026-02-18T06:52:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_42': 2026-02-18T06:52:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:52:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:52:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:52:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 147ff5c069b7 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.147ff5c069b7 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:50 2026 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:50 2026 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:50 2026 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:52:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 532bd8d2-fa6c-4c79-9f24-faec877f5667 2026-02-18T06:52:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:52:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:52:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-18T06:52:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:52:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:52:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T06:52:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:52:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T06:52:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:52:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:52:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 13650944 bytes 2026-02-18T06:52:51.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 29103104 bytes 2026-02-18T06:52:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 110357504 bytes 2026-02-18T06:52:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 158971904 bytes 2026-02-18T06:52:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 191682560 bytes 2026-02-18T06:52:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 332143616 bytes 2026-02-18T06:52:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 357066752 bytes 2026-02-18T06:52:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 400732160 bytes 2026-02-18T06:52:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 518706176 bytes 2026-02-18T06:52:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 599574528 bytes 2026-02-18T06:52:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 688503808 bytes 2026-02-18T06:52:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 748744704 bytes 2026-02-18T06:52:52.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 785531904 bytes 2026-02-18T06:52:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 830650368 bytes 2026-02-18T06:52:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 893763584 bytes 2026-02-18T06:52:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 1001967616 bytes 2026-02-18T06:52:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 1039104000 bytes 2026-02-18T06:52:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 1094046720 bytes 2026-02-18T06:52:52.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 1124600832 bytes 2026-02-18T06:52:52.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 1205837824 bytes 2026-02-18T06:52:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1231263744 bytes 2026-02-18T06:52:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1345775616 bytes 2026-02-18T06:52:52.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1412370432 bytes 2026-02-18T06:52:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1499665408 bytes 2026-02-18T06:52:53.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1557519360 bytes 2026-02-18T06:52:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1564399616 bytes 2026-02-18T06:52:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1613061120 bytes 2026-02-18T06:52:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1637315584 bytes 2026-02-18T06:52:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1696263168 bytes 2026-02-18T06:52:53.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1725213696 bytes 2026-02-18T06:52:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1820448768 bytes 2026-02-18T06:52:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1855915008 bytes 2026-02-18T06:52:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1940514816 bytes 2026-02-18T06:52:53.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1970352128 bytes 2026-02-18T06:52:53.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 2035113984 bytes 2026-02-18T06:52:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 2120631296 bytes 2026-02-18T06:52:53.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 2213264384 bytes 2026-02-18T06:52:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 2257808384 bytes 2026-02-18T06:52:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 2305648640 bytes 2026-02-18T06:52:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 2356568064 bytes 2026-02-18T06:52:54.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 2426094592 bytes 2026-02-18T06:52:54.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 2487003136 bytes 2026-02-18T06:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 2519944192 bytes 2026-02-18T06:52:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2563299328 bytes 2026-02-18T06:52:54.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2583166976 bytes 2026-02-18T06:52:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2641644544 bytes 2026-02-18T06:52:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2711431168 bytes 2026-02-18T06:52:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2792893440 bytes 2026-02-18T06:52:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2859790336 bytes 2026-02-18T06:52:54.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2964367360 bytes 2026-02-18T06:52:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 3033809920 bytes 2026-02-18T06:52:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 3038314496 bytes 2026-02-18T06:52:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 3094867968 bytes 2026-02-18T06:52:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 3142624256 bytes 2026-02-18T06:52:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 3181649920 bytes 2026-02-18T06:52:55.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 3186073600 bytes 2026-02-18T06:52:55.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 3190710272 bytes 2026-02-18T06:52:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 3227325440 bytes 2026-02-18T06:52:55.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 3250165760 bytes 2026-02-18T06:52:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 3278579712 bytes 2026-02-18T06:52:55.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 3279211520 bytes 2026-02-18T06:52:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 3297334272 bytes 2026-02-18T06:52:55.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 3320137728 bytes 2026-02-18T06:52:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 3418431488 bytes 2026-02-18T06:52:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3458124800 bytes 2026-02-18T06:52:55.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3479164928 bytes 2026-02-18T06:52:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3526389760 bytes 2026-02-18T06:52:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3599908864 bytes 2026-02-18T06:52:55.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3662464000 bytes 2026-02-18T06:52:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3679571968 bytes 2026-02-18T06:52:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3767932928 bytes 2026-02-18T06:52:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3822289920 bytes 2026-02-18T06:52:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3851012096 bytes 2026-02-18T06:52:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3972871168 bytes 2026-02-18T06:52:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 4032885760 bytes 2026-02-18T06:52:56.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 4050187264 bytes 2026-02-18T06:52:56.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 4057210880 bytes 2026-02-18T06:52:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 4091767808 bytes 2026-02-18T06:52:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 4180224000 bytes 2026-02-18T06:52:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 4206054400 bytes 2026-02-18T06:52:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 4257026048 bytes 2026-02-18T06:52:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 4298705920 bytes 2026-02-18T06:52:56.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 4342891520 bytes 2026-02-18T06:52:56.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 4387424256 bytes 2026-02-18T06:52:56.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 4397977600 bytes 2026-02-18T06:52:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4459044864 bytes 2026-02-18T06:52:57.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4492913664 bytes 2026-02-18T06:52:57.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4493530112 bytes 2026-02-18T06:52:57.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4566391808 bytes 2026-02-18T06:52:57.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4618327040 bytes 2026-02-18T06:52:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4642104320 bytes 2026-02-18T06:52:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4692795392 bytes 2026-02-18T06:52:57.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4789582848 bytes 2026-02-18T06:52:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4826196992 bytes 2026-02-18T06:52:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4869865472 bytes 2026-02-18T06:52:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4870453248 bytes 2026-02-18T06:52:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4935586816 bytes 2026-02-18T06:52:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4955590656 bytes 2026-02-18T06:52:57.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 5031260160 bytes 2026-02-18T06:52:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 5035906048 bytes 2026-02-18T06:52:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 5115261952 bytes 2026-02-18T06:52:58.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 5150601216 bytes 2026-02-18T06:52:58.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 5188881408 bytes 2026-02-18T06:52:58.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 5247523840 bytes 2026-02-18T06:52:58.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 5309432832 bytes 2026-02-18T06:52:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 5368968192 bytes 2026-02-18T06:52:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5385447424 bytes 2026-02-18T06:52:58.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5406021632 bytes 2026-02-18T06:52:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5413666816 bytes 2026-02-18T06:52:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5454381056 bytes 2026-02-18T06:52:58.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5495036928 bytes 2026-02-18T06:52:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5567860736 bytes 2026-02-18T06:52:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5652481024 bytes 2026-02-18T06:52:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5671923712 bytes 2026-02-18T06:52:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5733387264 bytes 2026-02-18T06:52:58.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5757440000 bytes 2026-02-18T06:52:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5851902976 bytes 2026-02-18T06:52:59.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5855612928 bytes 2026-02-18T06:52:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5893605376 bytes 2026-02-18T06:52:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5930303488 bytes 2026-02-18T06:52:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5982815232 bytes 2026-02-18T06:52:59.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 6014481408 bytes 2026-02-18T06:52:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 6036626432 bytes 2026-02-18T06:52:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 6102678528 bytes 2026-02-18T06:52:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 6152066048 bytes 2026-02-18T06:52:59.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 6152690688 bytes 2026-02-18T06:52:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 6195362816 bytes 2026-02-18T06:52:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 6278441984 bytes 2026-02-18T06:52:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 6318264320 bytes 2026-02-18T06:52:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 6379308032 bytes 2026-02-18T06:52:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 6454876160 bytes 2026-02-18T06:52:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 6479775744 bytes 2026-02-18T06:52:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6491747328 bytes 2026-02-18T06:53:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6519487488 bytes 2026-02-18T06:53:00.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6570218496 bytes 2026-02-18T06:53:00.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6611758080 bytes 2026-02-18T06:53:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6690011136 bytes 2026-02-18T06:53:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6701123584 bytes 2026-02-18T06:53:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6715102208 bytes 2026-02-18T06:53:00.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6779239424 bytes 2026-02-18T06:53:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6868911104 bytes 2026-02-18T06:53:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6892852224 bytes 2026-02-18T06:53:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6948398080 bytes 2026-02-18T06:53:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6971802624 bytes 2026-02-18T06:53:00.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 7025110016 bytes 2026-02-18T06:53:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 7070299136 bytes 2026-02-18T06:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 7076495360 bytes 2026-02-18T06:53:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 7115561984 bytes 2026-02-18T06:53:00.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 7200411648 bytes 2026-02-18T06:53:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 7254492160 bytes 2026-02-18T06:53:01.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 7329542144 bytes 2026-02-18T06:53:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 7354448896 bytes 2026-02-18T06:53:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 7374483456 bytes 2026-02-18T06:53:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 7462914048 bytes 2026-02-18T06:53:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 7488765952 bytes 2026-02-18T06:53:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 7508615168 bytes 2026-02-18T06:53:01.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7527150592 bytes 2026-02-18T06:53:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7603491840 bytes 2026-02-18T06:53:01.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7629212672 bytes 2026-02-18T06:53:01.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7700013056 bytes 2026-02-18T06:53:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7732298752 bytes 2026-02-18T06:53:01.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7806825472 bytes 2026-02-18T06:53:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7850504192 bytes 2026-02-18T06:53:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7883920384 bytes 2026-02-18T06:53:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7951148032 bytes 2026-02-18T06:53:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7999001600 bytes 2026-02-18T06:53:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 8077387776 bytes 2026-02-18T06:53:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 8100948992 bytes 2026-02-18T06:53:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 8142219264 bytes 2026-02-18T06:53:02.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 8155785216 bytes 2026-02-18T06:53:02.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 8198600704 bytes 2026-02-18T06:53:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 8219756544 bytes 2026-02-18T06:53:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 8245669888 bytes 2026-02-18T06:53:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 8319744000 bytes 2026-02-18T06:53:02.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 8381973504 bytes 2026-02-18T06:53:02.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 8393969664 bytes 2026-02-18T06:53:02.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 8423859200 bytes 2026-02-18T06:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 8432423936 bytes 2026-02-18T06:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 8444740608 bytes 2026-02-18T06:53:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 8519745536 bytes 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_43 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:03.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:03.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_43 2026-02-18T06:53:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2026-02-18T06:53:04.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:04.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_43': 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 148b15730f1e 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.148b15730f1e 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:03 2026 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:03 2026 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:03 2026 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9142b2d3-2190-43ee-9ae4-a23d350d7595 2026-02-18T06:53:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-18T06:53:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T06:53:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T06:53:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 77408256 bytes 2026-02-18T06:53:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 85332992 bytes 2026-02-18T06:53:04.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 143080448 bytes 2026-02-18T06:53:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 177360896 bytes 2026-02-18T06:53:04.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 256041984 bytes 2026-02-18T06:53:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 260285440 bytes 2026-02-18T06:53:04.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 286352384 bytes 2026-02-18T06:53:04.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 319830016 bytes 2026-02-18T06:53:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 331817984 bytes 2026-02-18T06:53:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 379225088 bytes 2026-02-18T06:53:04.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 470817792 bytes 2026-02-18T06:53:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 508343296 bytes 2026-02-18T06:53:04.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 529414144 bytes 2026-02-18T06:53:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 554764288 bytes 2026-02-18T06:53:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 623165440 bytes 2026-02-18T06:53:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 683604992 bytes 2026-02-18T06:53:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 764779520 bytes 2026-02-18T06:53:05.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 803828736 bytes 2026-02-18T06:53:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 807287808 bytes 2026-02-18T06:53:05.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 888344576 bytes 2026-02-18T06:53:05.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 912565248 bytes 2026-02-18T06:53:05.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 923711488 bytes 2026-02-18T06:53:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1011161088 bytes 2026-02-18T06:53:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1051607040 bytes 2026-02-18T06:53:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1072173056 bytes 2026-02-18T06:53:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1146617856 bytes 2026-02-18T06:53:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1202652160 bytes 2026-02-18T06:53:05.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1262788608 bytes 2026-02-18T06:53:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1294886912 bytes 2026-02-18T06:53:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1348269056 bytes 2026-02-18T06:53:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1383879680 bytes 2026-02-18T06:53:06.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1400603648 bytes 2026-02-18T06:53:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1453202432 bytes 2026-02-18T06:53:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1499269120 bytes 2026-02-18T06:53:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1533615104 bytes 2026-02-18T06:53:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1574066176 bytes 2026-02-18T06:53:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1616291840 bytes 2026-02-18T06:53:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1707331584 bytes 2026-02-18T06:53:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1714785280 bytes 2026-02-18T06:53:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1737488384 bytes 2026-02-18T06:53:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1844959232 bytes 2026-02-18T06:53:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1918746624 bytes 2026-02-18T06:53:06.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1956341760 bytes 2026-02-18T06:53:06.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 2015960064 bytes 2026-02-18T06:53:07.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2085019648 bytes 2026-02-18T06:53:07.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2149496832 bytes 2026-02-18T06:53:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2192389120 bytes 2026-02-18T06:53:07.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2266395648 bytes 2026-02-18T06:53:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2312905728 bytes 2026-02-18T06:53:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2345138176 bytes 2026-02-18T06:53:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2349971456 bytes 2026-02-18T06:53:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2431247360 bytes 2026-02-18T06:53:07.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2507742208 bytes 2026-02-18T06:53:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2508709888 bytes 2026-02-18T06:53:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2571948032 bytes 2026-02-18T06:53:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2651080704 bytes 2026-02-18T06:53:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2660788224 bytes 2026-02-18T06:53:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2730557440 bytes 2026-02-18T06:53:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2752699392 bytes 2026-02-18T06:53:08.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2754271232 bytes 2026-02-18T06:53:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2800803840 bytes 2026-02-18T06:53:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2850827264 bytes 2026-02-18T06:53:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2905184256 bytes 2026-02-18T06:53:08.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2978387968 bytes 2026-02-18T06:53:08.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 3003355136 bytes 2026-02-18T06:53:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3039415296 bytes 2026-02-18T06:53:08.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3113262080 bytes 2026-02-18T06:53:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3149660160 bytes 2026-02-18T06:53:08.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3194414080 bytes 2026-02-18T06:53:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3264907264 bytes 2026-02-18T06:53:08.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3276945408 bytes 2026-02-18T06:53:08.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3292450816 bytes 2026-02-18T06:53:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3335600128 bytes 2026-02-18T06:53:08.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3368502272 bytes 2026-02-18T06:53:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3432062976 bytes 2026-02-18T06:53:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3474038784 bytes 2026-02-18T06:53:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3535670272 bytes 2026-02-18T06:53:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3660827648 bytes 2026-02-18T06:53:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3703303168 bytes 2026-02-18T06:53:09.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3725826048 bytes 2026-02-18T06:53:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3795621888 bytes 2026-02-18T06:53:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3857990656 bytes 2026-02-18T06:53:09.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3918296064 bytes 2026-02-18T06:53:09.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 4066581504 bytes 2026-02-18T06:53:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 4109513728 bytes 2026-02-18T06:53:09.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4135477248 bytes 2026-02-18T06:53:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4223319040 bytes 2026-02-18T06:53:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4273160192 bytes 2026-02-18T06:53:10.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4293557248 bytes 2026-02-18T06:53:10.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4320011264 bytes 2026-02-18T06:53:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4327760896 bytes 2026-02-18T06:53:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4378963968 bytes 2026-02-18T06:53:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4462208000 bytes 2026-02-18T06:53:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4498771968 bytes 2026-02-18T06:53:10.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4588626944 bytes 2026-02-18T06:53:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4597634048 bytes 2026-02-18T06:53:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4661952512 bytes 2026-02-18T06:53:10.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4703127552 bytes 2026-02-18T06:53:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4831643648 bytes 2026-02-18T06:53:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4848685056 bytes 2026-02-18T06:53:10.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4892591104 bytes 2026-02-18T06:53:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4944438272 bytes 2026-02-18T06:53:11.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4982677504 bytes 2026-02-18T06:53:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 5043667968 bytes 2026-02-18T06:53:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 5078332416 bytes 2026-02-18T06:53:11.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 5115363328 bytes 2026-02-18T06:53:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5176101888 bytes 2026-02-18T06:53:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5260049408 bytes 2026-02-18T06:53:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5287932928 bytes 2026-02-18T06:53:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5367987200 bytes 2026-02-18T06:53:11.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5444718592 bytes 2026-02-18T06:53:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5445342208 bytes 2026-02-18T06:53:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5445954560 bytes 2026-02-18T06:53:11.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5481099264 bytes 2026-02-18T06:53:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5537707008 bytes 2026-02-18T06:53:11.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5567397888 bytes 2026-02-18T06:53:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5609052160 bytes 2026-02-18T06:53:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5644532736 bytes 2026-02-18T06:53:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5702818816 bytes 2026-02-18T06:53:12.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5743338496 bytes 2026-02-18T06:53:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5805196288 bytes 2026-02-18T06:53:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5814484992 bytes 2026-02-18T06:53:12.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5835452416 bytes 2026-02-18T06:53:12.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5870430208 bytes 2026-02-18T06:53:12.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5938097152 bytes 2026-02-18T06:53:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5970589696 bytes 2026-02-18T06:53:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5976549376 bytes 2026-02-18T06:53:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 6003034112 bytes 2026-02-18T06:53:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 6003647488 bytes 2026-02-18T06:53:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 6036527104 bytes 2026-02-18T06:53:12.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 6055911424 bytes 2026-02-18T06:53:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 6098851840 bytes 2026-02-18T06:53:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 6099451904 bytes 2026-02-18T06:53:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 6132001792 bytes 2026-02-18T06:53:12.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 6140081152 bytes 2026-02-18T06:53:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 6177427456 bytes 2026-02-18T06:53:13.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6268169216 bytes 2026-02-18T06:53:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6320462848 bytes 2026-02-18T06:53:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6420603904 bytes 2026-02-18T06:53:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6444055552 bytes 2026-02-18T06:53:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6482530304 bytes 2026-02-18T06:53:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6544155648 bytes 2026-02-18T06:53:13.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6604350464 bytes 2026-02-18T06:53:13.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6663891968 bytes 2026-02-18T06:53:13.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6743468032 bytes 2026-02-18T06:53:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6794700800 bytes 2026-02-18T06:53:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6803758080 bytes 2026-02-18T06:53:13.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6829647872 bytes 2026-02-18T06:53:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6896648192 bytes 2026-02-18T06:53:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6939314176 bytes 2026-02-18T06:53:14.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6963575808 bytes 2026-02-18T06:53:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6964178944 bytes 2026-02-18T06:53:14.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 7002737664 bytes 2026-02-18T06:53:14.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 7028822016 bytes 2026-02-18T06:53:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 7054085120 bytes 2026-02-18T06:53:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 7099262976 bytes 2026-02-18T06:53:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 7219559424 bytes 2026-02-18T06:53:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 7248335872 bytes 2026-02-18T06:53:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 7295557632 bytes 2026-02-18T06:53:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 7302854656 bytes 2026-02-18T06:53:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7347243008 bytes 2026-02-18T06:53:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7355069440 bytes 2026-02-18T06:53:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7456463872 bytes 2026-02-18T06:53:14.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7522399232 bytes 2026-02-18T06:53:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7533486080 bytes 2026-02-18T06:53:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7553382400 bytes 2026-02-18T06:53:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7613955072 bytes 2026-02-18T06:53:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7705968640 bytes 2026-02-18T06:53:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7747147776 bytes 2026-02-18T06:53:15.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7822488576 bytes 2026-02-18T06:53:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7867029504 bytes 2026-02-18T06:53:15.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7885316096 bytes 2026-02-18T06:53:15.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7888644096 bytes 2026-02-18T06:53:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7962174464 bytes 2026-02-18T06:53:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7999611904 bytes 2026-02-18T06:53:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 8006263808 bytes 2026-02-18T06:53:15.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 8013318144 bytes 2026-02-18T06:53:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 8073849856 bytes 2026-02-18T06:53:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 8083590144 bytes 2026-02-18T06:53:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 8163083264 bytes 2026-02-18T06:53:16.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 8193297408 bytes 2026-02-18T06:53:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 8236181504 bytes 2026-02-18T06:53:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 8254253056 bytes 2026-02-18T06:53:16.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 8271979520 bytes 2026-02-18T06:53:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8274012160 bytes 2026-02-18T06:53:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8297310208 bytes 2026-02-18T06:53:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8340457472 bytes 2026-02-18T06:53:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8404990976 bytes 2026-02-18T06:53:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8441820160 bytes 2026-02-18T06:53:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8484594688 bytes 2026-02-18T06:53:16.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8522698752 bytes 2026-02-18T06:53:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8573208576 bytes 2026-02-18T06:53:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8590562304 bytes 2026-02-18T06:53:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8610736128 bytes 2026-02-18T06:53:16.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8638395392 bytes 2026-02-18T06:53:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8676075520 bytes 2026-02-18T06:53:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8687357952 bytes 2026-02-18T06:53:16.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8689963008 bytes 2026-02-18T06:53:17.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8733642752 bytes 2026-02-18T06:53:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_44 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_44 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2026-02-18T06:53:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_44': 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 149640ccaa12 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.149640ccaa12 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:17 2026 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:17 2026 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:17 2026 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 711413a6-3d43-4895-ae70-a3f48d733603 2026-02-18T06:53:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T06:53:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:53:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 27185152 bytes 2026-02-18T06:53:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 63489024 bytes 2026-02-18T06:53:18.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 110452736 bytes 2026-02-18T06:53:18.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 168464384 bytes 2026-02-18T06:53:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 200185856 bytes 2026-02-18T06:53:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 242670592 bytes 2026-02-18T06:53:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 295199744 bytes 2026-02-18T06:53:18.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 319004672 bytes 2026-02-18T06:53:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 387931136 bytes 2026-02-18T06:53:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 440987648 bytes 2026-02-18T06:53:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 459210752 bytes 2026-02-18T06:53:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 520004608 bytes 2026-02-18T06:53:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 552402944 bytes 2026-02-18T06:53:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 577340416 bytes 2026-02-18T06:53:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 602475520 bytes 2026-02-18T06:53:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 610770944 bytes 2026-02-18T06:53:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 646650880 bytes 2026-02-18T06:53:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 680256512 bytes 2026-02-18T06:53:19.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 706729984 bytes 2026-02-18T06:53:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 788499456 bytes 2026-02-18T06:53:19.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 816983040 bytes 2026-02-18T06:53:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 842818560 bytes 2026-02-18T06:53:19.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 909792256 bytes 2026-02-18T06:53:19.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 973816832 bytes 2026-02-18T06:53:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 974451712 bytes 2026-02-18T06:53:19.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 991120384 bytes 2026-02-18T06:53:19.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1059085312 bytes 2026-02-18T06:53:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1063520256 bytes 2026-02-18T06:53:19.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1100434432 bytes 2026-02-18T06:53:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1112389632 bytes 2026-02-18T06:53:19.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1129936896 bytes 2026-02-18T06:53:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1155288064 bytes 2026-02-18T06:53:20.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1186863104 bytes 2026-02-18T06:53:20.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1220536320 bytes 2026-02-18T06:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1259106304 bytes 2026-02-18T06:53:20.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1292406784 bytes 2026-02-18T06:53:20.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1345014784 bytes 2026-02-18T06:53:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1354204160 bytes 2026-02-18T06:53:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1380020224 bytes 2026-02-18T06:53:20.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1448171520 bytes 2026-02-18T06:53:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1474546688 bytes 2026-02-18T06:53:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1504179200 bytes 2026-02-18T06:53:20.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1541804032 bytes 2026-02-18T06:53:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1542418432 bytes 2026-02-18T06:53:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1572422656 bytes 2026-02-18T06:53:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1601162240 bytes 2026-02-18T06:53:20.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1672744960 bytes 2026-02-18T06:53:20.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1703667712 bytes 2026-02-18T06:53:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1718398976 bytes 2026-02-18T06:53:20.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1764452352 bytes 2026-02-18T06:53:21.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1778008064 bytes 2026-02-18T06:53:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1806131200 bytes 2026-02-18T06:53:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1828089856 bytes 2026-02-18T06:53:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1916598272 bytes 2026-02-18T06:53:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 1989198848 bytes 2026-02-18T06:53:21.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2030891008 bytes 2026-02-18T06:53:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2060052480 bytes 2026-02-18T06:53:21.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2152413184 bytes 2026-02-18T06:53:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2208466944 bytes 2026-02-18T06:53:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2233255936 bytes 2026-02-18T06:53:21.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2260055040 bytes 2026-02-18T06:53:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2331548672 bytes 2026-02-18T06:53:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2332740608 bytes 2026-02-18T06:53:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2394848256 bytes 2026-02-18T06:53:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2476998656 bytes 2026-02-18T06:53:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2534536192 bytes 2026-02-18T06:53:22.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2592556032 bytes 2026-02-18T06:53:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2628904960 bytes 2026-02-18T06:53:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2669854720 bytes 2026-02-18T06:53:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2737269760 bytes 2026-02-18T06:53:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2758844416 bytes 2026-02-18T06:53:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2814619648 bytes 2026-02-18T06:53:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2867206144 bytes 2026-02-18T06:53:22.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2966596608 bytes 2026-02-18T06:53:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3048001536 bytes 2026-02-18T06:53:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3141666816 bytes 2026-02-18T06:53:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3231820800 bytes 2026-02-18T06:53:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3275363328 bytes 2026-02-18T06:53:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3338955776 bytes 2026-02-18T06:53:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3383484416 bytes 2026-02-18T06:53:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3422564352 bytes 2026-02-18T06:53:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3448113152 bytes 2026-02-18T06:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3499536384 bytes 2026-02-18T06:53:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3515494400 bytes 2026-02-18T06:53:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3516103680 bytes 2026-02-18T06:53:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3557057536 bytes 2026-02-18T06:53:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3644238848 bytes 2026-02-18T06:53:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3717264384 bytes 2026-02-18T06:53:23.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3760486400 bytes 2026-02-18T06:53:23.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3792833536 bytes 2026-02-18T06:53:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3818273792 bytes 2026-02-18T06:53:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3865487360 bytes 2026-02-18T06:53:23.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3866096640 bytes 2026-02-18T06:53:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3881739264 bytes 2026-02-18T06:53:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3899944960 bytes 2026-02-18T06:53:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3978379264 bytes 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_45 2026-02-18T06:53:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_45 2026-02-18T06:53:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2026-02-18T06:53:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2026-02-18T06:53:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_45': 2026-02-18T06:53:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14a2c2c7da0c 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14a2c2c7da0c 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:24 2026 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:24 2026 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:24 2026 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: f6f2ce66-9eb2-475c-ba7d-b2bc4c3e0d27 2026-02-18T06:53:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-18T06:53:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T06:53:25.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T06:53:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 25016320 bytes 2026-02-18T06:53:25.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 83268608 bytes 2026-02-18T06:53:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 136634368 bytes 2026-02-18T06:53:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 156483584 bytes 2026-02-18T06:53:25.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 157080576 bytes 2026-02-18T06:53:25.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 176719872 bytes 2026-02-18T06:53:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 215004160 bytes 2026-02-18T06:53:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 297197568 bytes 2026-02-18T06:53:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 342914048 bytes 2026-02-18T06:53:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 384925696 bytes 2026-02-18T06:53:25.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 462045184 bytes 2026-02-18T06:53:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 493554688 bytes 2026-02-18T06:53:25.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 543923200 bytes 2026-02-18T06:53:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 647887872 bytes 2026-02-18T06:53:26.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 678107136 bytes 2026-02-18T06:53:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 678728704 bytes 2026-02-18T06:53:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 716562432 bytes 2026-02-18T06:53:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 794993664 bytes 2026-02-18T06:53:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 822607872 bytes 2026-02-18T06:53:26.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 867562496 bytes 2026-02-18T06:53:26.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 868186112 bytes 2026-02-18T06:53:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 896395264 bytes 2026-02-18T06:53:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 949023744 bytes 2026-02-18T06:53:26.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 965408768 bytes 2026-02-18T06:53:26.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1035202560 bytes 2026-02-18T06:53:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1064295424 bytes 2026-02-18T06:53:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1087096832 bytes 2026-02-18T06:53:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1151713280 bytes 2026-02-18T06:53:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1171915776 bytes 2026-02-18T06:53:26.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1209140224 bytes 2026-02-18T06:53:27.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1259765760 bytes 2026-02-18T06:53:27.046 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:53:27.043+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f70964e0 2026-02-18T06:53:27.046 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:53:27.043+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f9092b60 2026-02-18T06:53:27.047 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:53:27.045+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f9dca4e0 2026-02-18T06:53:27.048 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:53:27.045+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fa363520 2026-02-18T06:53:27.048 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:53:27.045+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fb180410 2026-02-18T06:53:27.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1292932096 bytes 2026-02-18T06:53:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1352318976 bytes 2026-02-18T06:53:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1387651072 bytes 2026-02-18T06:53:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1419145216 bytes 2026-02-18T06:53:27.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1501282304 bytes 2026-02-18T06:53:27.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1503696896 bytes 2026-02-18T06:53:27.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1533829120 bytes 2026-02-18T06:53:27.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1558353920 bytes 2026-02-18T06:53:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1635294208 bytes 2026-02-18T06:53:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1707267072 bytes 2026-02-18T06:53:27.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1746161664 bytes 2026-02-18T06:53:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1836578816 bytes 2026-02-18T06:53:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1884111872 bytes 2026-02-18T06:53:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1908681728 bytes 2026-02-18T06:53:28.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1928809472 bytes 2026-02-18T06:53:28.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1996866560 bytes 2026-02-18T06:53:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2034250752 bytes 2026-02-18T06:53:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2088028160 bytes 2026-02-18T06:53:28.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2213904384 bytes 2026-02-18T06:53:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2271821824 bytes 2026-02-18T06:53:28.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2294481920 bytes 2026-02-18T06:53:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2354554880 bytes 2026-02-18T06:53:28.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2368163840 bytes 2026-02-18T06:53:28.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2381959168 bytes 2026-02-18T06:53:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2418626560 bytes 2026-02-18T06:53:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2451701760 bytes 2026-02-18T06:53:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2487823360 bytes 2026-02-18T06:53:29.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2555143168 bytes 2026-02-18T06:53:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2577601536 bytes 2026-02-18T06:53:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2615041024 bytes 2026-02-18T06:53:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2651484160 bytes 2026-02-18T06:53:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2688166912 bytes 2026-02-18T06:53:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2720152576 bytes 2026-02-18T06:53:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2827910144 bytes 2026-02-18T06:53:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2885580800 bytes 2026-02-18T06:53:29.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2965803008 bytes 2026-02-18T06:53:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3021089792 bytes 2026-02-18T06:53:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3081505792 bytes 2026-02-18T06:53:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3099711488 bytes 2026-02-18T06:53:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3130095616 bytes 2026-02-18T06:53:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3200225280 bytes 2026-02-18T06:53:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3219794944 bytes 2026-02-18T06:53:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3272774656 bytes 2026-02-18T06:53:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3295390720 bytes 2026-02-18T06:53:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3313417216 bytes 2026-02-18T06:53:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3382367232 bytes 2026-02-18T06:53:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3451828224 bytes 2026-02-18T06:53:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3462374400 bytes 2026-02-18T06:53:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3501878272 bytes 2026-02-18T06:53:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3527691264 bytes 2026-02-18T06:53:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3547087872 bytes 2026-02-18T06:53:30.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3566981120 bytes 2026-02-18T06:53:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3586847744 bytes 2026-02-18T06:53:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3587475456 bytes 2026-02-18T06:53:30.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3608715264 bytes 2026-02-18T06:53:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3636025344 bytes 2026-02-18T06:53:30.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3663720448 bytes 2026-02-18T06:53:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3699896320 bytes 2026-02-18T06:53:31.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3711426560 bytes 2026-02-18T06:53:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3756053504 bytes 2026-02-18T06:53:31.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3756680192 bytes 2026-02-18T06:53:31.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3778381824 bytes 2026-02-18T06:53:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3854825472 bytes 2026-02-18T06:53:31.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3901950976 bytes 2026-02-18T06:53:31.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3970563072 bytes 2026-02-18T06:53:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4054175744 bytes 2026-02-18T06:53:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4115909632 bytes 2026-02-18T06:53:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4184456192 bytes 2026-02-18T06:53:31.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4199544832 bytes 2026-02-18T06:53:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4219756544 bytes 2026-02-18T06:53:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4257967104 bytes 2026-02-18T06:53:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4297581568 bytes 2026-02-18T06:53:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4309353472 bytes 2026-02-18T06:53:31.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4341218304 bytes 2026-02-18T06:53:32.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4431623168 bytes 2026-02-18T06:53:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4510825472 bytes 2026-02-18T06:53:32.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4529240064 bytes 2026-02-18T06:53:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4627231744 bytes 2026-02-18T06:53:32.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4667555840 bytes 2026-02-18T06:53:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4695178240 bytes 2026-02-18T06:53:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4771726336 bytes 2026-02-18T06:53:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4837150720 bytes 2026-02-18T06:53:32.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4973831168 bytes 2026-02-18T06:53:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5011098624 bytes 2026-02-18T06:53:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5102649344 bytes 2026-02-18T06:53:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5143881728 bytes 2026-02-18T06:53:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5201222656 bytes 2026-02-18T06:53:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5246080000 bytes 2026-02-18T06:53:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5369690112 bytes 2026-02-18T06:53:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5482057728 bytes 2026-02-18T06:53:33.433 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5546830848 bytes 2026-02-18T06:53:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5603589120 bytes 2026-02-18T06:53:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5609914368 bytes 2026-02-18T06:53:33.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5646535680 bytes 2026-02-18T06:53:33.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5710598144 bytes 2026-02-18T06:53:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5725187072 bytes 2026-02-18T06:53:33.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5738891264 bytes 2026-02-18T06:53:33.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5797943296 bytes 2026-02-18T06:53:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5819254784 bytes 2026-02-18T06:53:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5851934720 bytes 2026-02-18T06:53:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_46 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_46 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2026-02-18T06:53:34.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T06:53:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2026-02-18T06:53:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2026-02-18T06:53:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_46': 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14aeada3fa82 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14aeada3fa82 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:34 2026 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:34 2026 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:34 2026 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9587dfb7-d60a-4ccf-8fb2-510f8d803efa 2026-02-18T06:53:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T06:53:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:53:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 34722816 bytes 2026-02-18T06:53:35.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 76751872 bytes 2026-02-18T06:53:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 86098944 bytes 2026-02-18T06:53:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 176944128 bytes 2026-02-18T06:53:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 234486784 bytes 2026-02-18T06:53:35.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 266203136 bytes 2026-02-18T06:53:35.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 273712128 bytes 2026-02-18T06:53:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 306434048 bytes 2026-02-18T06:53:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 349233152 bytes 2026-02-18T06:53:35.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 362076160 bytes 2026-02-18T06:53:35.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 402009088 bytes 2026-02-18T06:53:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 464285696 bytes 2026-02-18T06:53:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 510758912 bytes 2026-02-18T06:53:35.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 521675776 bytes 2026-02-18T06:53:36.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 618041344 bytes 2026-02-18T06:53:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 656860160 bytes 2026-02-18T06:53:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 700243968 bytes 2026-02-18T06:53:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 741532672 bytes 2026-02-18T06:53:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 804523008 bytes 2026-02-18T06:53:36.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 880249856 bytes 2026-02-18T06:53:36.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 935019520 bytes 2026-02-18T06:53:36.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 992937984 bytes 2026-02-18T06:53:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1113938944 bytes 2026-02-18T06:53:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1172071424 bytes 2026-02-18T06:53:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1200018432 bytes 2026-02-18T06:53:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1250220032 bytes 2026-02-18T06:53:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1297022976 bytes 2026-02-18T06:53:36.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1321233408 bytes 2026-02-18T06:53:36.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1344583680 bytes 2026-02-18T06:53:37.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1408157696 bytes 2026-02-18T06:53:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1444298752 bytes 2026-02-18T06:53:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1480044544 bytes 2026-02-18T06:53:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1515327488 bytes 2026-02-18T06:53:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1517713408 bytes 2026-02-18T06:53:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1531280384 bytes 2026-02-18T06:53:37.271 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7424208925180601) > weight (0.25) 2026-02-18T06:53:37.272 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3117590937596393) > weight (0.25) 2026-02-18T06:53:37.272 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T06:53:37.272 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T06:53:37.272 INFO:tasks.rbd_mirror_thrash:waiting for 1.3424655176577371 secs before reviving daemons 2026-02-18T06:53:37.273 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T06:53:37.270+0000 7f06e9fb3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 41588) UID: 1000 2026-02-18T06:53:37.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1562764288 bytes 2026-02-18T06:53:37.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1636626432 bytes 2026-02-18T06:53:37.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1688164352 bytes 2026-02-18T06:53:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1786866688 bytes 2026-02-18T06:53:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1812121600 bytes 2026-02-18T06:53:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1843993600 bytes 2026-02-18T06:53:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1860185088 bytes 2026-02-18T06:53:37.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1905055744 bytes 2026-02-18T06:53:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1929676800 bytes 2026-02-18T06:53:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1976712192 bytes 2026-02-18T06:53:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2008553472 bytes 2026-02-18T06:53:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2084603904 bytes 2026-02-18T06:53:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2105864192 bytes 2026-02-18T06:53:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2166644736 bytes 2026-02-18T06:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2224637952 bytes 2026-02-18T06:53:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2276583424 bytes 2026-02-18T06:53:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2311343104 bytes 2026-02-18T06:53:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2407835648 bytes 2026-02-18T06:53:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2486594560 bytes 2026-02-18T06:53:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2504877056 bytes 2026-02-18T06:53:38.616 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T06:53:38.616 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:53:38.617 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T06:53:38.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T06:53:38.618 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T06:53:38.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T06:53:38.621 INFO:tasks.rbd_mirror_thrash:waiting for 96.19740344744122 secs before thrashing 2026-02-18T06:53:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2565002240 bytes 2026-02-18T06:53:38.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2570765312 bytes 2026-02-18T06:53:38.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2666717184 bytes 2026-02-18T06:53:38.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2730881024 bytes 2026-02-18T06:53:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2793143296 bytes 2026-02-18T06:53:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2821353472 bytes 2026-02-18T06:53:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2842146816 bytes 2026-02-18T06:53:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2883180544 bytes 2026-02-18T06:53:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2935745536 bytes 2026-02-18T06:53:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2982660096 bytes 2026-02-18T06:53:39.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3025598464 bytes 2026-02-18T06:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3074536448 bytes 2026-02-18T06:53:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3082141696 bytes 2026-02-18T06:53:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3120884736 bytes 2026-02-18T06:53:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3162671104 bytes 2026-02-18T06:53:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3217275904 bytes 2026-02-18T06:53:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3249566720 bytes 2026-02-18T06:53:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3310684160 bytes 2026-02-18T06:53:39.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3328871424 bytes 2026-02-18T06:53:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3387968512 bytes 2026-02-18T06:53:39.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3437942784 bytes 2026-02-18T06:53:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3471901696 bytes 2026-02-18T06:53:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3524242432 bytes 2026-02-18T06:53:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3568653312 bytes 2026-02-18T06:53:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3628983296 bytes 2026-02-18T06:53:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3674798080 bytes 2026-02-18T06:53:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3763021824 bytes 2026-02-18T06:53:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3805371392 bytes 2026-02-18T06:53:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3822635008 bytes 2026-02-18T06:53:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3850773504 bytes 2026-02-18T06:53:40.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3887248384 bytes 2026-02-18T06:53:40.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3935047680 bytes 2026-02-18T06:53:40.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3990482944 bytes 2026-02-18T06:53:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4055493632 bytes 2026-02-18T06:53:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4060086272 bytes 2026-02-18T06:53:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4143197184 bytes 2026-02-18T06:53:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4241380352 bytes 2026-02-18T06:53:41.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4316029952 bytes 2026-02-18T06:53:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4389703680 bytes 2026-02-18T06:53:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4470971392 bytes 2026-02-18T06:53:41.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4565298176 bytes 2026-02-18T06:53:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4615994368 bytes 2026-02-18T06:53:41.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4694376448 bytes 2026-02-18T06:53:41.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4722996224 bytes 2026-02-18T06:53:41.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4748538880 bytes 2026-02-18T06:53:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4804736000 bytes 2026-02-18T06:53:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4846120960 bytes 2026-02-18T06:53:41.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4918251520 bytes 2026-02-18T06:53:41.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4926610432 bytes 2026-02-18T06:53:41.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4950311936 bytes 2026-02-18T06:53:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4966837248 bytes 2026-02-18T06:53:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_47 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_47 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T06:53:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2026-02-18T06:53:43.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2026-02-18T06:53:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_47': 2026-02-18T06:53:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:43.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14c031df93b3 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14c031df93b3 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:42 2026 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:42 2026 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:42 2026 2026-02-18T06:53:43.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 8a85931f-5e1b-4cb0-8882-4ef6fe35af93 2026-02-18T06:53:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T06:53:43.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:53:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:43.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:43.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 49432576 bytes 2026-02-18T06:53:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 62057472 bytes 2026-02-18T06:53:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 105821184 bytes 2026-02-18T06:53:43.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 126869504 bytes 2026-02-18T06:53:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 173600768 bytes 2026-02-18T06:53:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 238951424 bytes 2026-02-18T06:53:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 286036992 bytes 2026-02-18T06:53:43.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 305939456 bytes 2026-02-18T06:53:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 363507712 bytes 2026-02-18T06:53:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 467020800 bytes 2026-02-18T06:53:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 487265280 bytes 2026-02-18T06:53:43.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 557618176 bytes 2026-02-18T06:53:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 621936640 bytes 2026-02-18T06:53:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 658092032 bytes 2026-02-18T06:53:44.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 693584896 bytes 2026-02-18T06:53:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 747656192 bytes 2026-02-18T06:53:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 780969984 bytes 2026-02-18T06:53:44.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 829522944 bytes 2026-02-18T06:53:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 885175296 bytes 2026-02-18T06:53:44.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 897227776 bytes 2026-02-18T06:53:44.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 949366784 bytes 2026-02-18T06:53:44.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 987173888 bytes 2026-02-18T06:53:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 993419264 bytes 2026-02-18T06:53:44.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1004140544 bytes 2026-02-18T06:53:44.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1037937664 bytes 2026-02-18T06:53:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1052453888 bytes 2026-02-18T06:53:45.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1085785088 bytes 2026-02-18T06:53:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1113431040 bytes 2026-02-18T06:53:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1189753856 bytes 2026-02-18T06:53:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1282957312 bytes 2026-02-18T06:53:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1334755328 bytes 2026-02-18T06:53:45.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1372112896 bytes 2026-02-18T06:53:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1390383104 bytes 2026-02-18T06:53:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1448960000 bytes 2026-02-18T06:53:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1495084032 bytes 2026-02-18T06:53:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1533316096 bytes 2026-02-18T06:53:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1577421824 bytes 2026-02-18T06:53:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1640165376 bytes 2026-02-18T06:53:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1662481408 bytes 2026-02-18T06:53:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1699903488 bytes 2026-02-18T06:53:45.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1712748544 bytes 2026-02-18T06:53:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1772638208 bytes 2026-02-18T06:53:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1857354752 bytes 2026-02-18T06:53:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1891608576 bytes 2026-02-18T06:53:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1975925760 bytes 2026-02-18T06:53:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2033025024 bytes 2026-02-18T06:53:46.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2053739520 bytes 2026-02-18T06:53:46.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2103153664 bytes 2026-02-18T06:53:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2146562048 bytes 2026-02-18T06:53:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2232250368 bytes 2026-02-18T06:53:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2291038208 bytes 2026-02-18T06:53:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2344086528 bytes 2026-02-18T06:53:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2360767488 bytes 2026-02-18T06:53:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2379083776 bytes 2026-02-18T06:53:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2422479872 bytes 2026-02-18T06:53:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2495404032 bytes 2026-02-18T06:53:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2519234560 bytes 2026-02-18T06:53:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2584886272 bytes 2026-02-18T06:53:47.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2619867136 bytes 2026-02-18T06:53:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2680586240 bytes 2026-02-18T06:53:47.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2684224512 bytes 2026-02-18T06:53:47.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2709109760 bytes 2026-02-18T06:53:47.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2760437760 bytes 2026-02-18T06:53:47.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2775797760 bytes 2026-02-18T06:53:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2796084224 bytes 2026-02-18T06:53:47.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2811101184 bytes 2026-02-18T06:53:47.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2852603904 bytes 2026-02-18T06:53:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2958047232 bytes 2026-02-18T06:53:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2978587648 bytes 2026-02-18T06:53:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3004129280 bytes 2026-02-18T06:53:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3011871744 bytes 2026-02-18T06:53:48.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3096918016 bytes 2026-02-18T06:53:48.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3142467584 bytes 2026-02-18T06:53:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3204988928 bytes 2026-02-18T06:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3271310336 bytes 2026-02-18T06:53:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3319917568 bytes 2026-02-18T06:53:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3359422464 bytes 2026-02-18T06:53:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3405945856 bytes 2026-02-18T06:53:48.511 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3408112640 bytes 2026-02-18T06:53:48.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3446447104 bytes 2026-02-18T06:53:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3516617728 bytes 2026-02-18T06:53:48.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3558181888 bytes 2026-02-18T06:53:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3576392704 bytes 2026-02-18T06:53:48.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3618716672 bytes 2026-02-18T06:53:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3686630400 bytes 2026-02-18T06:53:48.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3712005120 bytes 2026-02-18T06:53:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3714078720 bytes 2026-02-18T06:53:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_48 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_48 2026-02-18T06:53:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:49.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2026-02-18T06:53:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2026-02-18T06:53:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_48': 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14ccbbd4d44e 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14ccbbd4d44e 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:49 2026 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:49 2026 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:49 2026 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 0f8c4865-0f0d-4a82-ba86-812a3b418b6a 2026-02-18T06:53:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T06:53:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=8 2026-02-18T06:53:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 66732032 bytes 2026-02-18T06:53:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 112842752 bytes 2026-02-18T06:53:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 183525376 bytes 2026-02-18T06:53:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 295186432 bytes 2026-02-18T06:53:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 342702080 bytes 2026-02-18T06:53:50.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 397335552 bytes 2026-02-18T06:53:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 435468288 bytes 2026-02-18T06:53:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 459270144 bytes 2026-02-18T06:53:50.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 484644864 bytes 2026-02-18T06:53:50.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 518319104 bytes 2026-02-18T06:53:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 572492800 bytes 2026-02-18T06:53:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 614796288 bytes 2026-02-18T06:53:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 628009984 bytes 2026-02-18T06:53:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 723384320 bytes 2026-02-18T06:53:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 808296448 bytes 2026-02-18T06:53:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 844036096 bytes 2026-02-18T06:53:51.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 869520384 bytes 2026-02-18T06:53:51.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 925937664 bytes 2026-02-18T06:53:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 938752000 bytes 2026-02-18T06:53:51.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 969592832 bytes 2026-02-18T06:53:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 994881536 bytes 2026-02-18T06:53:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1078680576 bytes 2026-02-18T06:53:51.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1148502016 bytes 2026-02-18T06:53:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1275100160 bytes 2026-02-18T06:53:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1287071744 bytes 2026-02-18T06:53:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1287681024 bytes 2026-02-18T06:53:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1288903680 bytes 2026-02-18T06:53:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1350355968 bytes 2026-02-18T06:53:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1395040256 bytes 2026-02-18T06:53:52.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1425475584 bytes 2026-02-18T06:53:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1447010304 bytes 2026-02-18T06:53:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1472293888 bytes 2026-02-18T06:53:52.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1491674112 bytes 2026-02-18T06:53:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1497848832 bytes 2026-02-18T06:53:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1564732416 bytes 2026-02-18T06:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1606255616 bytes 2026-02-18T06:53:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1702221824 bytes 2026-02-18T06:53:52.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1723112448 bytes 2026-02-18T06:53:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1751408640 bytes 2026-02-18T06:53:52.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1779444736 bytes 2026-02-18T06:53:52.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1829019648 bytes 2026-02-18T06:53:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1851301888 bytes 2026-02-18T06:53:52.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1911690240 bytes 2026-02-18T06:53:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1939986432 bytes 2026-02-18T06:53:52.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2019267584 bytes 2026-02-18T06:53:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2042655744 bytes 2026-02-18T06:53:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2096568320 bytes 2026-02-18T06:53:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2135264256 bytes 2026-02-18T06:53:53.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2135884800 bytes 2026-02-18T06:53:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2230987776 bytes 2026-02-18T06:53:53.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2268293120 bytes 2026-02-18T06:53:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2335232000 bytes 2026-02-18T06:53:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2425393152 bytes 2026-02-18T06:53:53.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2460812288 bytes 2026-02-18T06:53:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2517580800 bytes 2026-02-18T06:53:53.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2591599616 bytes 2026-02-18T06:53:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2641944576 bytes 2026-02-18T06:53:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2686098432 bytes 2026-02-18T06:53:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2705833984 bytes 2026-02-18T06:53:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2723871744 bytes 2026-02-18T06:53:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2735722496 bytes 2026-02-18T06:53:54.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2750031872 bytes 2026-02-18T06:53:54.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2835843072 bytes 2026-02-18T06:53:54.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2941464576 bytes 2026-02-18T06:53:54.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2972093440 bytes 2026-02-18T06:53:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 3011717120 bytes 2026-02-18T06:53:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 3039556608 bytes 2026-02-18T06:53:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3059680256 bytes 2026-02-18T06:53:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3085249536 bytes 2026-02-18T06:53:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3108079616 bytes 2026-02-18T06:53:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3151834112 bytes 2026-02-18T06:53:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3170603008 bytes 2026-02-18T06:53:54.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3214558208 bytes 2026-02-18T06:53:54.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3259596800 bytes 2026-02-18T06:53:54.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3326494720 bytes 2026-02-18T06:53:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3371374592 bytes 2026-02-18T06:53:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3456769024 bytes 2026-02-18T06:53:55.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3499199488 bytes 2026-02-18T06:53:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3557997568 bytes 2026-02-18T06:53:55.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3603489792 bytes 2026-02-18T06:53:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3678027776 bytes 2026-02-18T06:53:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3747214336 bytes 2026-02-18T06:53:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3852647424 bytes 2026-02-18T06:53:55.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3900729344 bytes 2026-02-18T06:53:55.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3933365248 bytes 2026-02-18T06:53:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 4015423488 bytes 2026-02-18T06:53:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 4016046080 bytes 2026-02-18T06:53:55.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 4060877824 bytes 2026-02-18T06:53:55.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4120412160 bytes 2026-02-18T06:53:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4171268096 bytes 2026-02-18T06:53:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4190114816 bytes 2026-02-18T06:53:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4244274176 bytes 2026-02-18T06:53:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4267725824 bytes 2026-02-18T06:53:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4330542080 bytes 2026-02-18T06:53:56.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4352338944 bytes 2026-02-18T06:53:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4412659712 bytes 2026-02-18T06:53:56.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4442494976 bytes 2026-02-18T06:53:56.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4527706112 bytes 2026-02-18T06:53:56.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4598490112 bytes 2026-02-18T06:53:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4636364800 bytes 2026-02-18T06:53:56.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4685277184 bytes 2026-02-18T06:53:56.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4762870784 bytes 2026-02-18T06:53:56.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4868478976 bytes 2026-02-18T06:53:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4895326208 bytes 2026-02-18T06:53:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4968694784 bytes 2026-02-18T06:53:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 5048929280 bytes 2026-02-18T06:53:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 5065351168 bytes 2026-02-18T06:53:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 5088165888 bytes 2026-02-18T06:53:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 5141808128 bytes 2026-02-18T06:53:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 5179570176 bytes 2026-02-18T06:53:57.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 5215088640 bytes 2026-02-18T06:53:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5240268800 bytes 2026-02-18T06:53:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5333794816 bytes 2026-02-18T06:53:57.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5400586240 bytes 2026-02-18T06:53:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5408959488 bytes 2026-02-18T06:53:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5474163712 bytes 2026-02-18T06:53:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5529354240 bytes 2026-02-18T06:53:57.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5550699520 bytes 2026-02-18T06:53:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5592836096 bytes 2026-02-18T06:53:57.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5629740032 bytes 2026-02-18T06:53:58.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5645506560 bytes 2026-02-18T06:53:58.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5656686592 bytes 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_49 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T06:53:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:53:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:53:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:53:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:53:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_49 2026-02-18T06:53:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:53:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:53:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2026-02-18T06:53:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:53:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2026-02-18T06:53:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_49': 2026-02-18T06:53:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:53:59.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14d89d11d275 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14d89d11d275 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:58 2026 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:58 2026 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:58 2026 2026-02-18T06:53:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:53:59.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:53:59.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9f47bbaa-cf2e-4ab0-887a-190da2abbcb4 2026-02-18T06:53:59.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T06:53:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:53:59.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T06:53:59.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:53:59.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:53:59.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 18985984 bytes 2026-02-18T06:53:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 40370176 bytes 2026-02-18T06:53:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 74869760 bytes 2026-02-18T06:53:59.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 150376448 bytes 2026-02-18T06:53:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 175514624 bytes 2026-02-18T06:53:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 206204928 bytes 2026-02-18T06:53:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 322354176 bytes 2026-02-18T06:53:59.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 391969792 bytes 2026-02-18T06:53:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 394492928 bytes 2026-02-18T06:53:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 395124736 bytes 2026-02-18T06:53:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 436856832 bytes 2026-02-18T06:53:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 495876096 bytes 2026-02-18T06:53:59.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 508548096 bytes 2026-02-18T06:53:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 572765184 bytes 2026-02-18T06:54:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 626713600 bytes 2026-02-18T06:54:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 700768256 bytes 2026-02-18T06:54:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 758333440 bytes 2026-02-18T06:54:00.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 793269248 bytes 2026-02-18T06:54:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 819251200 bytes 2026-02-18T06:54:00.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 843604992 bytes 2026-02-18T06:54:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 904193024 bytes 2026-02-18T06:54:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 960162816 bytes 2026-02-18T06:54:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 978667520 bytes 2026-02-18T06:54:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1023390720 bytes 2026-02-18T06:54:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1121971200 bytes 2026-02-18T06:54:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1178109952 bytes 2026-02-18T06:54:00.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1189115904 bytes 2026-02-18T06:54:00.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1229363200 bytes 2026-02-18T06:54:00.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1272453120 bytes 2026-02-18T06:54:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1336506368 bytes 2026-02-18T06:54:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1355130880 bytes 2026-02-18T06:54:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1411182592 bytes 2026-02-18T06:54:01.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1458989056 bytes 2026-02-18T06:54:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1491665920 bytes 2026-02-18T06:54:01.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1531618304 bytes 2026-02-18T06:54:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1606948864 bytes 2026-02-18T06:54:01.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1660892160 bytes 2026-02-18T06:54:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1662602240 bytes 2026-02-18T06:54:01.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1665525760 bytes 2026-02-18T06:54:01.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1711197184 bytes 2026-02-18T06:54:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1766140928 bytes 2026-02-18T06:54:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1770654720 bytes 2026-02-18T06:54:01.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1794265088 bytes 2026-02-18T06:54:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1820910592 bytes 2026-02-18T06:54:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1851492352 bytes 2026-02-18T06:54:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1905382400 bytes 2026-02-18T06:54:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1916695552 bytes 2026-02-18T06:54:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1980519424 bytes 2026-02-18T06:54:02.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2022157312 bytes 2026-02-18T06:54:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2058342400 bytes 2026-02-18T06:54:02.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2090975232 bytes 2026-02-18T06:54:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2154218496 bytes 2026-02-18T06:54:02.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2190904320 bytes 2026-02-18T06:54:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2224667648 bytes 2026-02-18T06:54:02.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2230309888 bytes 2026-02-18T06:54:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2293984256 bytes 2026-02-18T06:54:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2365566976 bytes 2026-02-18T06:54:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2381029376 bytes 2026-02-18T06:54:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2395090944 bytes 2026-02-18T06:54:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2440850432 bytes 2026-02-18T06:54:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2494246912 bytes 2026-02-18T06:54:02.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2519713792 bytes 2026-02-18T06:54:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2563437568 bytes 2026-02-18T06:54:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2583834624 bytes 2026-02-18T06:54:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2638020608 bytes 2026-02-18T06:54:03.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2657589248 bytes 2026-02-18T06:54:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2698258432 bytes 2026-02-18T06:54:03.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2744891392 bytes 2026-02-18T06:54:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2768002048 bytes 2026-02-18T06:54:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2799679488 bytes 2026-02-18T06:54:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2833072128 bytes 2026-02-18T06:54:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2866558976 bytes 2026-02-18T06:54:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2873518080 bytes 2026-02-18T06:54:03.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2932417536 bytes 2026-02-18T06:54:03.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2990765056 bytes 2026-02-18T06:54:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3035762688 bytes 2026-02-18T06:54:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3054712832 bytes 2026-02-18T06:54:03.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3079318528 bytes 2026-02-18T06:54:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3131851776 bytes 2026-02-18T06:54:03.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3164913664 bytes 2026-02-18T06:54:03.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3229573120 bytes 2026-02-18T06:54:04.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3283734528 bytes 2026-02-18T06:54:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3326727168 bytes 2026-02-18T06:54:04.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3355802624 bytes 2026-02-18T06:54:04.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3433849856 bytes 2026-02-18T06:54:04.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3497198592 bytes 2026-02-18T06:54:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3543074816 bytes 2026-02-18T06:54:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3590512640 bytes 2026-02-18T06:54:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3615348736 bytes 2026-02-18T06:54:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3742054400 bytes 2026-02-18T06:54:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3799997440 bytes 2026-02-18T06:54:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3886818304 bytes 2026-02-18T06:54:05.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3928906752 bytes 2026-02-18T06:54:05.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4013127680 bytes 2026-02-18T06:54:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4075086848 bytes 2026-02-18T06:54:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4122253312 bytes 2026-02-18T06:54:05.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4193605632 bytes 2026-02-18T06:54:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4231687168 bytes 2026-02-18T06:54:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4312562688 bytes 2026-02-18T06:54:05.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4384635904 bytes 2026-02-18T06:54:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4396986368 bytes 2026-02-18T06:54:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4431702016 bytes 2026-02-18T06:54:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4462486528 bytes 2026-02-18T06:54:05.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4520198144 bytes 2026-02-18T06:54:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4568316928 bytes 2026-02-18T06:54:05.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4609043456 bytes 2026-02-18T06:54:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4632259584 bytes 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_50 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n snapshot ']' 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 128M ']' 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=128M 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T06:54:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering mirror/image_50 2026-02-18T06:54:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ is_pool_mirror_mode_image mirror 2026-02-18T06:54:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' image = image ']' 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T06:54:06.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2026-02-18T06:54:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T06:54:07.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2026-02-18T06:54:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_50': 2026-02-18T06:54:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:54:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 14e4a54ce647 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.14e4a54ce647 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:54:06 2026 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:54:06 2026 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:54:06 2026 2026-02-18T06:54:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:54:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:54:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: efa032aa-a7f7-4e9f-a922-c3940dfa3a3f 2026-02-18T06:54:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n '' ']' 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T06:54:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T06:54:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=6 2026-02-18T06:54:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T06:54:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-18T06:54:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 17920000 bytes 2026-02-18T06:54:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 48603136 bytes 2026-02-18T06:54:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 78338048 bytes 2026-02-18T06:54:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 103154688 bytes 2026-02-18T06:54:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 144217088 bytes 2026-02-18T06:54:07.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 184691712 bytes 2026-02-18T06:54:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 224994304 bytes 2026-02-18T06:54:07.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 304661504 bytes 2026-02-18T06:54:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 330201088 bytes 2026-02-18T06:54:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 383879168 bytes 2026-02-18T06:54:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 504208384 bytes 2026-02-18T06:54:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 577600512 bytes 2026-02-18T06:54:08.019 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T06:54:08.019 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T06:54:08.019 INFO:tasks.rbd_mirror_thrash:waiting for 7.266402878360866 secs before reviving daemons 2026-02-18T06:54:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 636569600 bytes 2026-02-18T06:54:08.019 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T06:54:08.018+0000 7f6260b35640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 39491) UID: 1000 2026-02-18T06:54:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 672363520 bytes 2026-02-18T06:54:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 719965184 bytes 2026-02-18T06:54:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 759587840 bytes 2026-02-18T06:54:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 779208704 bytes 2026-02-18T06:54:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 791497728 bytes 2026-02-18T06:54:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 830305280 bytes 2026-02-18T06:54:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 830950400 bytes 2026-02-18T06:54:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 903693312 bytes 2026-02-18T06:54:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 918118400 bytes 2026-02-18T06:54:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 953346048 bytes 2026-02-18T06:54:08.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1024460800 bytes 2026-02-18T06:54:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1149895680 bytes 2026-02-18T06:54:08.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1175362560 bytes 2026-02-18T06:54:08.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1228090368 bytes 2026-02-18T06:54:08.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1290422272 bytes 2026-02-18T06:54:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1332963328 bytes 2026-02-18T06:54:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1398366208 bytes 2026-02-18T06:54:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1407997952 bytes 2026-02-18T06:54:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1453701120 bytes 2026-02-18T06:54:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1455404032 bytes 2026-02-18T06:54:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1492076544 bytes 2026-02-18T06:54:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1493481472 bytes 2026-02-18T06:54:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1536617472 bytes 2026-02-18T06:54:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1643027456 bytes 2026-02-18T06:54:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1690735616 bytes 2026-02-18T06:54:09.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1771600896 bytes 2026-02-18T06:54:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1818417152 bytes 2026-02-18T06:54:09.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1862328320 bytes 2026-02-18T06:54:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1893349376 bytes 2026-02-18T06:54:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1927271424 bytes 2026-02-18T06:54:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1956393984 bytes 2026-02-18T06:54:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2006642688 bytes 2026-02-18T06:54:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2040899584 bytes 2026-02-18T06:54:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2075817984 bytes 2026-02-18T06:54:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2087732224 bytes 2026-02-18T06:54:10.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2144712704 bytes 2026-02-18T06:54:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2196979712 bytes 2026-02-18T06:54:10.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2218802176 bytes 2026-02-18T06:54:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2259879936 bytes 2026-02-18T06:54:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2301113344 bytes 2026-02-18T06:54:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2321052672 bytes 2026-02-18T06:54:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2335115264 bytes 2026-02-18T06:54:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2428052480 bytes 2026-02-18T06:54:10.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2515095552 bytes 2026-02-18T06:54:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2564929536 bytes 2026-02-18T06:54:11.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2575513600 bytes 2026-02-18T06:54:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2598084608 bytes 2026-02-18T06:54:11.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2656752640 bytes 2026-02-18T06:54:11.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2712283136 bytes 2026-02-18T06:54:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2812377088 bytes 2026-02-18T06:54:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2868701184 bytes 2026-02-18T06:54:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2895703040 bytes 2026-02-18T06:54:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2916571136 bytes 2026-02-18T06:54:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2994761728 bytes 2026-02-18T06:54:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3079482368 bytes 2026-02-18T06:54:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3130834944 bytes 2026-02-18T06:54:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3143644160 bytes 2026-02-18T06:54:11.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3158801408 bytes 2026-02-18T06:54:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3232907264 bytes 2026-02-18T06:54:11.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3265452032 bytes 2026-02-18T06:54:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3309296640 bytes 2026-02-18T06:54:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3322602496 bytes 2026-02-18T06:54:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3383457792 bytes 2026-02-18T06:54:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3395634176 bytes 2026-02-18T06:54:12.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3452306432 bytes 2026-02-18T06:54:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3483022336 bytes 2026-02-18T06:54:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3537515520 bytes 2026-02-18T06:54:12.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3620718592 bytes 2026-02-18T06:54:12.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3647507456 bytes 2026-02-18T06:54:12.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3678838784 bytes 2026-02-18T06:54:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3710316544 bytes 2026-02-18T06:54:12.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3729004544 bytes 2026-02-18T06:54:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3782607872 bytes 2026-02-18T06:54:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3858978816 bytes 2026-02-18T06:54:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3916125184 bytes 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:13.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_count=50 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local count 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_count=0 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:13.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T06:54:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T06:54:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T06:54:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-18T06:54:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'images: ' 2026-02-18T06:54:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 2 2026-02-18T06:54:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ count=50 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 50 = 50 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state 2026-02-18T06:54:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T06:54:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T06:54:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T06:54:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-18T06:54:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'image health:' 2026-02-18T06:54:13.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 3 2026-02-18T06:54:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T06:54:13.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=OK 2026-02-18T06:54:13.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = ERROR 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = OK 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_count=50 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local count 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_count=0 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:13.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T06:54:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T06:54:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T06:54:13.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T06:54:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'images: ' 2026-02-18T06:54:13.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 2 2026-02-18T06:54:13.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T06:54:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ count=50 2026-02-18T06:54:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 50 = 50 2026-02-18T06:54:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-18T06:54:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state 2026-02-18T06:54:13.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T06:54:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T06:54:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T06:54:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T06:54:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'image health:' 2026-02-18T06:54:13.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 3 2026-02-18T06:54:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T06:54:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=WARNING 2026-02-18T06:54:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test WARNING = ERROR 2026-02-18T06:54:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test WARNING = OK 2026-02-18T06:54:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T06:54:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 2 -ne 1 2026-02-18T06:54:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 30 2026-02-18T06:54:15.286 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T06:54:15.287 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:54:15.287 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T06:54:15.287 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T06:54:15.288 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T06:54:15.290 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T06:54:15.291 INFO:tasks.rbd_mirror_thrash:waiting for 64.78664194168063 secs before thrashing 2026-02-18T06:54:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T06:54:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'image health:' 2026-02-18T06:54:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 3 2026-02-18T06:54:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T06:54:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=OK 2026-02-18T06:54:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = ERROR 2026-02-18T06:54:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = OK 2026-02-18T06:54:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: compare many images' 2026-02-18T06:54:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T06:54:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T06:54:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 06:54:43 TEST: compare many images 2026-02-18T06:54:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 06:54:43 TEST: compare many images 2026-02-18T06:54:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 50 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_1 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:54:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-18T06:54:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:54:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:54:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:54:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:54:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:54:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:54:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-18T06:54:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 1 closest matches: 2026-02-18T06:54:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_18 2026-02-18T06:54:44.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:54:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:54:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:44.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2026-02-18T06:54:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 3 closest matches: 2026-02-18T06:54:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_12 2026-02-18T06:54:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_13 2026-02-18T06:54:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_19 2026-02-18T06:54:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:54:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:54:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2026-02-18T06:54:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:54:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:54:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:54:44.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:44.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2026-02-18T06:54:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 87 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_1 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_1.status 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_1.status 2026-02-18T06:54:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:45.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.status 2026-02-18T06:54:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_1.status 2026-02-18T06:54:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_1.status 2026-02-18T06:54:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.status 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.status 2026-02-18T06:54:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2026-02-18T06:54:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:54:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=87 2026-02-18T06:54:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-18T06:54:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:54:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 87 = '' 2026-02-18T06:54:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:54:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-18T06:54:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:46.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 87 = '' 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:54:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.status 2026-02-18T06:54:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-18T06:54:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=87 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 87 = 87 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:54:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-18T06:54:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_1': 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b73bfc7b13 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b73bfc7b13 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:54:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:45:39 2026 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:45:39 2026 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:45:39 2026 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1f218227-0cf6-470c-8df3-a73ed823e755 2026-02-18T06:54:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:54:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-18T06:54:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.export 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.export 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.export 2026-02-18T06:54:47.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.export 2026-02-18T06:54:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:54:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.export 2026-02-18T06:54:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:54:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.export 2026-02-18T06:54:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_1.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_1.export 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_2 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-18T06:54:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:54:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:54:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:54:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:54:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:54:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-18T06:54:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:54:50.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:54:50.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:54:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-18T06:54:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 4 closest matches: 2026-02-18T06:54:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_22 2026-02-18T06:54:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_24 2026-02-18T06:54:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_26 2026-02-18T06:54:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_28 2026-02-18T06:54:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-18T06:54:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:54:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:54:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-18T06:54:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 3 closest matches: 2026-02-18T06:54:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_20 2026-02-18T06:54:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_23 2026-02-18T06:54:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_27 2026-02-18T06:54:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:54:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:54:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:54:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:54:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-18T06:54:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-18T06:54:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:54:50.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_2 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:54:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:54:50.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:54:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:54:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:54:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:54:50.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_2 2026-02-18T06:54:50.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:54:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:54:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:54:50.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:54:50.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:54:50.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2026-02-18T06:54:50.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:50.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2026-02-18T06:54:51.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 89 2026-02-18T06:54:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_2 2026-02-18T06:54:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_2.status 2026-02-18T06:54:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_2.status 2026-02-18T06:54:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.status 2026-02-18T06:54:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_2.status 2026-02-18T06:54:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_2.status 2026-02-18T06:54:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.status 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.status 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2026-02-18T06:54:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:54:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=89 2026-02-18T06:54:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-18T06:54:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:51.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=36 2026-02-18T06:54:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 89 = 36 2026-02-18T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-18T06:54:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:54:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:54:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 89 = '' 2026-02-18T06:54:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:54:52.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-18T06:54:52.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:54:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:54:52.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:54:52.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:54:52.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 89 = '' 2026-02-18T06:54:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.status 2026-02-18T06:54:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-18T06:54:54.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:54.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=89 2026-02-18T06:54:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 89 = 89 2026-02-18T06:54:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:54:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:55.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_2': 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11919ba15f25 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11919ba15f25 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:54:55.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:45:54 2026 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:45:54 2026 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:45:54 2026 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e6c1e58d-7558-4bf6-9061-94218ae350a1 2026-02-18T06:54:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:54:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.export 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.export 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.export 2026-02-18T06:54:55.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.export 2026-02-18T06:54:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:54:55.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.export 2026-02-18T06:54:55.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:54:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.export 2026-02-18T06:54:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_2.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_2.export 2026-02-18T06:54:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:54:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_3 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:54:56.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:54:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:54:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:54:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:54:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:54:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:54:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:54:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:54:57.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-18T06:54:57.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 2 closest matches: 2026-02-18T06:54:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_34 2026-02-18T06:54:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_38 2026-02-18T06:54:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:54:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:54:57.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:57.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:54:57.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:54:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:54:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:57.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:54:57.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-02-18T06:54:57.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:54:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:54:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:54:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:54:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:54:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:54:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:54:57.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2026-02-18T06:54:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 91 2026-02-18T06:54:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_3 2026-02-18T06:54:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:54:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:54:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:54:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_3.status 2026-02-18T06:54:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_3.status 2026-02-18T06:54:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:57.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.status 2026-02-18T06:54:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_3.status 2026-02-18T06:54:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_3.status 2026-02-18T06:54:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:54:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.status 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.status 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2026-02-18T06:54:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:57.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:54:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=91 2026-02-18T06:54:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-18T06:54:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=37 2026-02-18T06:54:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 91 = 37 2026-02-18T06:54:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:57.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-18T06:54:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:54:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:54:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:54:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:54:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:54:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 91 = '' 2026-02-18T06:54:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:54:57.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:54:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.status 2026-02-18T06:54:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-18T06:54:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:54:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=91 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 91 = 91 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:54:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:54:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-18T06:54:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_3': 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11be4bed4f1e 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11be4bed4f1e 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:08 2026 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:08 2026 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:08 2026 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: ef7fee7b-f8b3-4ef1-ade2-f49cb926b8d3 2026-02-18T06:54:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.export 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.export 2026-02-18T06:54:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.export 2026-02-18T06:54:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.export 2026-02-18T06:54:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:54:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.export 2026-02-18T06:55:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:00.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.export 2026-02-18T06:55:00.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_3.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_3.export 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_4 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:02.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-18T06:55:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2026-02-18T06:55:03.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 93 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_4 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_4.status 2026-02-18T06:55:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_4.status 2026-02-18T06:55:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.status 2026-02-18T06:55:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_4.status 2026-02-18T06:55:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_4.status 2026-02-18T06:55:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.status 2026-02-18T06:55:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.status 2026-02-18T06:55:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2026-02-18T06:55:03.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:03.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=93 2026-02-18T06:55:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-18T06:55:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:03.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=38 2026-02-18T06:55:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 93 = 38 2026-02-18T06:55:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:03.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-18T06:55:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=38 2026-02-18T06:55:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 93 = 38 2026-02-18T06:55:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-18T06:55:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 93 = '' 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:06.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.status 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-18T06:55:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=93 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 93 = 93 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:06.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:06.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-18T06:55:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_4': 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5a94fcb93 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5a94fcb93 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:20 2026 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:20 2026 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:20 2026 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e7796cb1-112a-4c40-81fa-5cc2bae63d79 2026-02-18T06:55:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:07.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:07.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-18T06:55:07.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.export 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.export 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.export 2026-02-18T06:55:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.export 2026-02-18T06:55:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:07.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.export 2026-02-18T06:55:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.export 2026-02-18T06:55:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_4.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_4.export 2026-02-18T06:55:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_5 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-18T06:55:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:08.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:08.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:08.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:08.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:08.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-18T06:55:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:no valid command found; 1 closest matches: 2026-02-18T06:55:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status mirror/image_50 2026-02-18T06:55:09.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:09.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:09.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:09.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:09.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-18T06:55:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-18T06:55:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-18T06:55:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2026-02-18T06:55:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:09.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2026-02-18T06:55:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 95 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_5 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:09.985 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:09.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_5.status 2026-02-18T06:55:09.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_5.status 2026-02-18T06:55:09.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:09.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.status 2026-02-18T06:55:09.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_5.status 2026-02-18T06:55:09.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_5.status 2026-02-18T06:55:09.987 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.status 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.status 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2026-02-18T06:55:09.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=95 2026-02-18T06:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-18T06:55:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:10.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=39 2026-02-18T06:55:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 95 = 39 2026-02-18T06:55:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-18T06:55:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=39 2026-02-18T06:55:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 95 = 39 2026-02-18T06:55:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-18T06:55:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 95 = '' 2026-02-18T06:55:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.status 2026-02-18T06:55:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-18T06:55:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:13.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=95 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 95 = 95 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:13.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:14.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-18T06:55:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_5': 2026-02-18T06:55:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b73528142c 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b73528142c 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:34 2026 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:34 2026 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:34 2026 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: db0ba49e-c26a-4f8a-bc0c-a15701fe2493 2026-02-18T06:55:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.export 2026-02-18T06:55:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.export 2026-02-18T06:55:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.export 2026-02-18T06:55:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.export 2026-02-18T06:55:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:14.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.export 2026-02-18T06:55:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.export 2026-02-18T06:55:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_5.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_5.export 2026-02-18T06:55:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_6 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:14.819 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2803105857907098) > weight (0.25) 2026-02-18T06:55:14.819 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.967421802577163) > weight (0.25) 2026-02-18T06:55:14.819 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6534830152199426) > weight (0.25) 2026-02-18T06:55:14.819 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7681088730985324) > weight (0.25) 2026-02-18T06:55:14.819 INFO:tasks.rbd_mirror_thrash:waiting for 93.99931579936717 secs before thrashing 2026-02-18T06:55:15.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:16.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-18T06:55:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T06:55:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-18T06:55:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-18T06:55:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-18T06:55:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_6 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_6 2026-02-18T06:55:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:16.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2026-02-18T06:55:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2026-02-18T06:55:17.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 97 2026-02-18T06:55:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_6 2026-02-18T06:55:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_6.status 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_6.status 2026-02-18T06:55:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.status 2026-02-18T06:55:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_6.status 2026-02-18T06:55:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_6.status 2026-02-18T06:55:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.status 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.status 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2026-02-18T06:55:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=97 2026-02-18T06:55:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:17.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-18T06:55:17.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:17.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=40 2026-02-18T06:55:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 97 = 40 2026-02-18T06:55:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-18T06:55:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=40 2026-02-18T06:55:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 97 = 40 2026-02-18T06:55:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:18.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:18.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-18T06:55:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:18.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:18.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 97 = '' 2026-02-18T06:55:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:18.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:20.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6404933857362495) > weight (0.25) 2026-02-18T06:55:20.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7300124857060617) > weight (0.25) 2026-02-18T06:55:20.079 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.25750309213930433) > weight (0.25) 2026-02-18T06:55:20.079 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7969926306916364) > weight (0.25) 2026-02-18T06:55:20.079 INFO:tasks.rbd_mirror_thrash:waiting for 77.37901869732815 secs before thrashing 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.status 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-18T06:55:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=97 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 97 = 97 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_6': 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1260f09d8659 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1260f09d8659 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:46:47 2026 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:46:47 2026 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:46:47 2026 2026-02-18T06:55:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 42f0c001-dae4-46f5-8bba-6fe81293309d 2026-02-18T06:55:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.export 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.export 2026-02-18T06:55:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.export 2026-02-18T06:55:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.export 2026-02-18T06:55:21.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:21.445 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.export 2026-02-18T06:55:21.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.export 2026-02-18T06:55:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_6.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_6.export 2026-02-18T06:55:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_7 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:21.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:22.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:23.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:23.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-18T06:55:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-18T06:55:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-02-18T06:55:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:23.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:23.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:23.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:23.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:23.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:23.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:23.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-02-18T06:55:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-18T06:55:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2026-02-18T06:55:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2026-02-18T06:55:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:23.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:23.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2026-02-18T06:55:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2026-02-18T06:55:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 99 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_7 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_7.status 2026-02-18T06:55:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_7.status 2026-02-18T06:55:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.status 2026-02-18T06:55:24.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_7.status 2026-02-18T06:55:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_7.status 2026-02-18T06:55:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.status 2026-02-18T06:55:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.status 2026-02-18T06:55:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2026-02-18T06:55:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:24.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=99 2026-02-18T06:55:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:24.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:24.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-18T06:55:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:24.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=41 2026-02-18T06:55:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 99 = 41 2026-02-18T06:55:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-18T06:55:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:24.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=41 2026-02-18T06:55:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 99 = 41 2026-02-18T06:55:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-18T06:55:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:25.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:25.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:25.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:25.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:25.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:25.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:25.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 99 = '' 2026-02-18T06:55:25.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:25.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.status 2026-02-18T06:55:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-18T06:55:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=99 2026-02-18T06:55:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 99 = 99 2026-02-18T06:55:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-18T06:55:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_7': 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11916746fd4e 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11916746fd4e 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:02 2026 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:02 2026 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:02 2026 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 21e2d0de-af6e-45ad-b644-f4eee184726c 2026-02-18T06:55:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.export 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.export 2026-02-18T06:55:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.export 2026-02-18T06:55:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.export 2026-02-18T06:55:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.export 2026-02-18T06:55:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:28.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.export 2026-02-18T06:55:28.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_7.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_7.export 2026-02-18T06:55:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_8 2026-02-18T06:55:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-18T06:55:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:28.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-18T06:55:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-18T06:55:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-18T06:55:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:30.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2026-02-18T06:55:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:30.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2026-02-18T06:55:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-18T06:55:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_8 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:30.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_8 2026-02-18T06:55:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2026-02-18T06:55:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 101 2026-02-18T06:55:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_8 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_8.status 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_8.status 2026-02-18T06:55:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.status 2026-02-18T06:55:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_8.status 2026-02-18T06:55:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_8.status 2026-02-18T06:55:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.status 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.status 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2026-02-18T06:55:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=101 2026-02-18T06:55:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:31.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-18T06:55:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:31.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:31.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=42 2026-02-18T06:55:31.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 101 = 42 2026-02-18T06:55:31.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:31.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:31.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-18T06:55:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:31.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=42 2026-02-18T06:55:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 101 = 42 2026-02-18T06:55:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:32.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-18T06:55:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 101 = '' 2026-02-18T06:55:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.status 2026-02-18T06:55:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-18T06:55:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=101 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 101 = 101 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:35.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-18T06:55:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_8': 2026-02-18T06:55:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5f7dfd73f 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5f7dfd73f 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:08 2026 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:08 2026 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:08 2026 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 19f68516-8b2a-45d7-b3fb-465c882305ae 2026-02-18T06:55:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.export 2026-02-18T06:55:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.export 2026-02-18T06:55:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.export 2026-02-18T06:55:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.export 2026-02-18T06:55:35.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:35.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.export 2026-02-18T06:55:35.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:35.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.export 2026-02-18T06:55:35.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_8.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_8.export 2026-02-18T06:55:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_9 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-18T06:55:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:36.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:37.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:37.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-18T06:55:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T06:55:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:37.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:37.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:37.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:37.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-18T06:55:37.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-18T06:55:37.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:37.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:37.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-02-18T06:55:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2026-02-18T06:55:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 103 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_9 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_9.status 2026-02-18T06:55:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_9.status 2026-02-18T06:55:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.status 2026-02-18T06:55:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_9.status 2026-02-18T06:55:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_9.status 2026-02-18T06:55:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.status 2026-02-18T06:55:38.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.status 2026-02-18T06:55:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2026-02-18T06:55:38.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=103 2026-02-18T06:55:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-18T06:55:38.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=43 2026-02-18T06:55:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 103 = 43 2026-02-18T06:55:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:38.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-18T06:55:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:38.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=43 2026-02-18T06:55:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 103 = 43 2026-02-18T06:55:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:38.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-18T06:55:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:39.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 103 = '' 2026-02-18T06:55:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:39.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.status 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-18T06:55:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=103 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 103 = 103 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_9': 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 10b7d21b7e7e 2026-02-18T06:55:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.10b7d21b7e7e 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:24 2026 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:24 2026 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:24 2026 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 40b4e69c-a609-443c-8267-78314c9f7989 2026-02-18T06:55:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:42.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.export 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.export 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.export 2026-02-18T06:55:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.export 2026-02-18T06:55:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.export 2026-02-18T06:55:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.export 2026-02-18T06:55:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_9.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_9.export 2026-02-18T06:55:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_10 2026-02-18T06:55:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-18T06:55:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-18T06:55:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-18T06:55:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-18T06:55:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:44.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-18T06:55:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T06:55:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-18T06:55:44.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T06:55:44.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:44.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-02-18T06:55:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-02-18T06:55:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-18T06:55:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:44.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:55:44.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:55:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:55:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:55:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:55:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:55:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-02-18T06:55:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:44.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:44.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:44.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2026-02-18T06:55:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 105 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_10 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_10.status 2026-02-18T06:55:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_10.status 2026-02-18T06:55:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.status 2026-02-18T06:55:45.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_10.status 2026-02-18T06:55:45.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_10.status 2026-02-18T06:55:45.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.status 2026-02-18T06:55:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.status 2026-02-18T06:55:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2026-02-18T06:55:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:45.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=105 2026-02-18T06:55:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-18T06:55:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=44 2026-02-18T06:55:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 105 = 44 2026-02-18T06:55:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-18T06:55:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=44 2026-02-18T06:55:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 105 = 44 2026-02-18T06:55:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-18T06:55:46.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:46.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:46.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:46.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 105 = '' 2026-02-18T06:55:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:46.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.status 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-18T06:55:48.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=105 2026-02-18T06:55:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 105 = 105 2026-02-18T06:55:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:48.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_10': 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1191d443602b 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1191d443602b 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:36 2026 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:36 2026 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:36 2026 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c38dfa61-f880-425c-89ec-99dda99f916d 2026-02-18T06:55:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.export 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.export 2026-02-18T06:55:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.export 2026-02-18T06:55:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.export 2026-02-18T06:55:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.export 2026-02-18T06:55:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.export 2026-02-18T06:55:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_10.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_10.export 2026-02-18T06:55:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_11 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-18T06:55:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:50.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:50.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:50.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-18T06:55:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T06:55:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:51.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:51.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-18T06:55:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-18T06:55:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:55:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:55:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:55:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:55:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-18T06:55:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:51.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:51.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2026-02-18T06:55:51.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 107 2026-02-18T06:55:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_11 2026-02-18T06:55:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_11.status 2026-02-18T06:55:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_11.status 2026-02-18T06:55:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.status 2026-02-18T06:55:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_11.status 2026-02-18T06:55:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_11.status 2026-02-18T06:55:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.status 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.status 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2026-02-18T06:55:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=107 2026-02-18T06:55:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:52.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-18T06:55:52.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=45 2026-02-18T06:55:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 107 = 45 2026-02-18T06:55:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:52.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-18T06:55:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:52.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:52.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=45 2026-02-18T06:55:52.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 107 = 45 2026-02-18T06:55:52.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-18T06:55:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:55:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:55:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:55:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:55:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 107 = '' 2026-02-18T06:55:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:55:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.status 2026-02-18T06:55:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-18T06:55:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:55.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=107 2026-02-18T06:55:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 107 = 107 2026-02-18T06:55:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:55:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-18T06:55:56.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_11': 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5bd6c5512 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5bd6c5512 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:46 2026 2026-02-18T06:55:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:46 2026 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:46 2026 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 4a7b60e8-59db-4352-8976-1ffb0ad77ed8 2026-02-18T06:55:56.147 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:55:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-18T06:55:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T06:55:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:55:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.export 2026-02-18T06:55:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.export 2026-02-18T06:55:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.export 2026-02-18T06:55:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.export 2026-02-18T06:55:56.217 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:55:56.215+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fb181a00 2026-02-18T06:55:56.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.export 2026-02-18T06:55:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:55:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.export 2026-02-18T06:55:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_11.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_11.export 2026-02-18T06:55:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_12 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:55:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:55:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-18T06:55:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:55:58.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:55:58.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:55:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:55:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:55:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:55:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:55:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-18T06:55:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-18T06:55:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:55:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:55:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:55:58.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:55:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:55:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:55:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:55:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:55:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-18T06:55:58.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:55:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:55:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:55:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:55:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2026-02-18T06:55:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 109 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_12 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_12.status 2026-02-18T06:55:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_12.status 2026-02-18T06:55:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.status 2026-02-18T06:55:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_12.status 2026-02-18T06:55:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_12.status 2026-02-18T06:55:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:55:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:55:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.status 2026-02-18T06:55:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:55:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.status 2026-02-18T06:55:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2026-02-18T06:55:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:55:59.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=109 2026-02-18T06:55:59.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:55:59.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:59.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:55:59.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-18T06:55:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=46 2026-02-18T06:55:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 109 = 46 2026-02-18T06:55:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-18T06:55:59.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:55:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:55:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=46 2026-02-18T06:55:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 109 = 46 2026-02-18T06:55:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:55:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:56:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:56:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:56:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-18T06:56:00.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:00.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:00.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:00.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:00.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:00.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 109 = '' 2026-02-18T06:56:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:02.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:56:02.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:56:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.status 2026-02-18T06:56:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-18T06:56:02.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=109 2026-02-18T06:56:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 109 = 109 2026-02-18T06:56:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-18T06:56:02.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-18T06:56:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_12': 2026-02-18T06:56:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12602616fd02 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12602616fd02 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:47:54 2026 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:47:54 2026 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:47:54 2026 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: e98f19fc-6436-4e93-bab9-e22e39c2f301 2026-02-18T06:56:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.export 2026-02-18T06:56:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.export 2026-02-18T06:56:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.export 2026-02-18T06:56:03.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.export 2026-02-18T06:56:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.export 2026-02-18T06:56:03.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:03.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.export 2026-02-18T06:56:03.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_12.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_12.export 2026-02-18T06:56:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_13 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:03.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-18T06:56:04.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-18T06:56:04.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:04.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:04.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-18T06:56:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-18T06:56:04.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:04.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:04.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-18T06:56:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:04.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2026-02-18T06:56:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 111 2026-02-18T06:56:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_13 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_13.status 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_13.status 2026-02-18T06:56:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.status 2026-02-18T06:56:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_13.status 2026-02-18T06:56:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_13.status 2026-02-18T06:56:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.status 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.status 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2026-02-18T06:56:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=111 2026-02-18T06:56:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-18T06:56:06.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 111 = '' 2026-02-18T06:56:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-18T06:56:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:06.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:06.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:06.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 111 = '' 2026-02-18T06:56:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:06.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.status 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-18T06:56:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:07.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=111 2026-02-18T06:56:07.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 111 = 111 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:07.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-18T06:56:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_13': 2026-02-18T06:56:08.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126e59472b74 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126e59472b74 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:05 2026 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:05 2026 2026-02-18T06:56:08.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:05 2026 2026-02-18T06:56:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: ac0bdee8-83d2-49fa-8584-10a7ba48c5e6 2026-02-18T06:56:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.export 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.export 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.export 2026-02-18T06:56:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.export 2026-02-18T06:56:08.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:08.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.export 2026-02-18T06:56:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:08.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.export 2026-02-18T06:56:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_13.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_13.export 2026-02-18T06:56:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_14 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-18T06:56:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-18T06:56:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:10.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:10.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-18T06:56:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T06:56:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:10.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:10.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:10.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-18T06:56:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-18T06:56:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:56:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:56:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:56:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-18T06:56:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2026-02-18T06:56:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 113 2026-02-18T06:56:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_14 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_14.status 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_14.status 2026-02-18T06:56:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.status 2026-02-18T06:56:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_14.status 2026-02-18T06:56:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_14.status 2026-02-18T06:56:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:11.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:11.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.status 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.status 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2026-02-18T06:56:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:11.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=113 2026-02-18T06:56:11.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:11.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-18T06:56:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=48 2026-02-18T06:56:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 113 = 48 2026-02-18T06:56:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-18T06:56:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:11.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=48 2026-02-18T06:56:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 113 = 48 2026-02-18T06:56:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:12.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:12.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-18T06:56:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 113 = '' 2026-02-18T06:56:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.status 2026-02-18T06:56:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-18T06:56:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:14.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=113 2026-02-18T06:56:14.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 113 = 113 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-18T06:56:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_14': 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126e1cab1da8 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126e1cab1da8 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:15.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:13 2026 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:13 2026 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:13 2026 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 28c18533-c7df-40a3-9c10-0a9ba74d304a 2026-02-18T06:56:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.export 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.export 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.export 2026-02-18T06:56:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.export 2026-02-18T06:56:15.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.export 2026-02-18T06:56:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:15.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.export 2026-02-18T06:56:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_14.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_14.export 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_15 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:17.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-18T06:56:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T06:56:17.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:17.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-18T06:56:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:17.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:17.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-18T06:56:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T06:56:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:56:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:17.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:56:17.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:56:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:56:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2026-02-18T06:56:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-18T06:56:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_15 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:17.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:56:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:56:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:56:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:56:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:56:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:56:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:56:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_15 2026-02-18T06:56:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2026-02-18T06:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 115 2026-02-18T06:56:18.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_15 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_15.status 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_15.status 2026-02-18T06:56:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.status 2026-02-18T06:56:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_15.status 2026-02-18T06:56:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_15.status 2026-02-18T06:56:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.status 2026-02-18T06:56:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:18.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.status 2026-02-18T06:56:18.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2026-02-18T06:56:18.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=115 2026-02-18T06:56:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-18T06:56:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=49 2026-02-18T06:56:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 115 = 49 2026-02-18T06:56:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-18T06:56:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 115 = '' 2026-02-18T06:56:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:19.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-18T06:56:19.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:20.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:20.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:20.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 115 = '' 2026-02-18T06:56:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.status 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-18T06:56:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=115 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 115 = 115 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_15': 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126e4f14930d 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126e4f14930d 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:27 2026 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:27 2026 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:27 2026 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 35047dfb-3412-4601-ba30-18dbf9de2aaf 2026-02-18T06:56:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.export 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.export 2026-02-18T06:56:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.export 2026-02-18T06:56:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.export 2026-02-18T06:56:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:22.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.export 2026-02-18T06:56:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.export 2026-02-18T06:56:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_15.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_15.export 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_16 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-18T06:56:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:24.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:24.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:25.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-18T06:56:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T06:56:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-18T06:56:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-18T06:56:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:56:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:56:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-18T06:56:25.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2026-02-18T06:56:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2026-02-18T06:56:26.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 117 2026-02-18T06:56:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_16 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_16.status 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_16.status 2026-02-18T06:56:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.status 2026-02-18T06:56:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_16.status 2026-02-18T06:56:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_16.status 2026-02-18T06:56:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.status 2026-02-18T06:56:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.status 2026-02-18T06:56:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2026-02-18T06:56:26.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=117 2026-02-18T06:56:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-18T06:56:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:26.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=50 2026-02-18T06:56:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 117 = 50 2026-02-18T06:56:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:26.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:26.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-18T06:56:26.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:26.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=50 2026-02-18T06:56:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 117 = 50 2026-02-18T06:56:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-18T06:56:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 117 = '' 2026-02-18T06:56:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.status 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-18T06:56:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=117 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 117 = 117 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-18T06:56:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_16': 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126ef62963f2 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126ef62963f2 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:34 2026 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:34 2026 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:34 2026 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c8eb0f42-d61b-4ba7-ad8e-859cc232365a 2026-02-18T06:56:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.export 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.export 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.export 2026-02-18T06:56:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.export 2026-02-18T06:56:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.export 2026-02-18T06:56:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.export 2026-02-18T06:56:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_16.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_16.export 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_17 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-18T06:56:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:32.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:32.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-18T06:56:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:32.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-18T06:56:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T06:56:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-02-18T06:56:32.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-18T06:56:32.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:56:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:56:32.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:56:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:56:32.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2026-02-18T06:56:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2026-02-18T06:56:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2026-02-18T06:56:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 119 2026-02-18T06:56:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_17 2026-02-18T06:56:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_17.status 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_17.status 2026-02-18T06:56:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.status 2026-02-18T06:56:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_17.status 2026-02-18T06:56:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_17.status 2026-02-18T06:56:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.status 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.status 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2026-02-18T06:56:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:33.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=119 2026-02-18T06:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-18T06:56:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=51 2026-02-18T06:56:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 119 = 51 2026-02-18T06:56:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-18T06:56:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:33.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=51 2026-02-18T06:56:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 119 = 51 2026-02-18T06:56:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:33.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-18T06:56:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 119 = '' 2026-02-18T06:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:34.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.status 2026-02-18T06:56:36.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-18T06:56:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=119 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 119 = 119 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-18T06:56:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_17': 2026-02-18T06:56:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:37.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1191d34baf50 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1191d34baf50 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:41 2026 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:41 2026 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:41 2026 2026-02-18T06:56:37.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b71799a9-93c2-42d8-b47e-5494b730ec39 2026-02-18T06:56:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-18T06:56:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.export 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.export 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.export 2026-02-18T06:56:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.export 2026-02-18T06:56:37.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.export 2026-02-18T06:56:37.458 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6645587191284124) > weight (0.25) 2026-02-18T06:56:37.458 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9616727651908624) > weight (0.25) 2026-02-18T06:56:37.459 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4259185455242379) > weight (0.25) 2026-02-18T06:56:37.459 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.48594586007188123) > weight (0.25) 2026-02-18T06:56:37.459 INFO:tasks.rbd_mirror_thrash:waiting for 100.53101987701639 secs before thrashing 2026-02-18T06:56:37.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.export 2026-02-18T06:56:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_17.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_17.export 2026-02-18T06:56:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_18 2026-02-18T06:56:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-18T06:56:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:38.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:39.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-18T06:56:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:39.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:39.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:39.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2026-02-18T06:56:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2026-02-18T06:56:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 121 2026-02-18T06:56:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_18 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_18.status 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_18.status 2026-02-18T06:56:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.status 2026-02-18T06:56:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_18.status 2026-02-18T06:56:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_18.status 2026-02-18T06:56:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.status 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.status 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2026-02-18T06:56:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=121 2026-02-18T06:56:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-18T06:56:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=52 2026-02-18T06:56:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 121 = 52 2026-02-18T06:56:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-18T06:56:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=52 2026-02-18T06:56:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 121 = 52 2026-02-18T06:56:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:41.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-18T06:56:41.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:41.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:41.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:41.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 121 = '' 2026-02-18T06:56:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:43.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.status 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-18T06:56:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=121 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 121 = 121 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:56:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:44.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-18T06:56:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_18': 2026-02-18T06:56:44.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f518754e47 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f518754e47 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:51 2026 2026-02-18T06:56:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:51 2026 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:51 2026 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 58feaf8b-9b70-499d-a121-13c9d6bfc92a 2026-02-18T06:56:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:56:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-18T06:56:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.export 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.export 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.export 2026-02-18T06:56:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.export 2026-02-18T06:56:44.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.export 2026-02-18T06:56:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:56:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.export 2026-02-18T06:56:44.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_18.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_18.export 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_19 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:56:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:56:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:56:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:56:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-18T06:56:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:56:46.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:46.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:56:46.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:56:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:56:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-18T06:56:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-18T06:56:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:56:46.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:56:46.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:56:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:56:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:56:46.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-02-18T06:56:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:56:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:56:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:56:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:56:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2026-02-18T06:56:47.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 123 2026-02-18T06:56:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_19 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_19.status 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_19.status 2026-02-18T06:56:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.status 2026-02-18T06:56:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_19.status 2026-02-18T06:56:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_19.status 2026-02-18T06:56:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.status 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.status 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:56:47.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=123 2026-02-18T06:56:47.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:47.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:47.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:47.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=53 2026-02-18T06:56:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = 53 2026-02-18T06:56:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=53 2026-02-18T06:56:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = 53 2026-02-18T06:56:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:56:48.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:48.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:56:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:56:48.819 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7024225804707088) > weight (0.25) 2026-02-18T06:56:48.819 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T06:56:48.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T06:56:48.819 INFO:tasks.rbd_mirror_thrash:waiting for 5.49212982503899 secs before reviving daemons 2026-02-18T06:56:48.819 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:56:48.819+0000 7f17f5126640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 39291) UID: 1000 2026-02-18T06:56:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:50.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:50.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:50.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:56:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:52.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:56:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T06:56:54.312 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T06:56:54.312 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:56:54.312 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T06:56:54.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T06:56:54.312 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T06:56:54.315 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T06:56:54.315 INFO:tasks.rbd_mirror_thrash:waiting for 64.34021660822428 secs before thrashing 2026-02-18T06:56:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:54.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:54.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:54.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:54.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:54.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:56:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:54.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T06:56:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:56:58.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:56:58.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:56:58.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:56:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:56:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:56:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:56:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:56:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:56:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:56:58.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T06:57:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:57:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:02.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T06:57:10.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:10.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:57:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:57:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T06:57:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:57:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:18.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = '' 2026-02-18T06:57:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T06:57:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.status 2026-02-18T06:57:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-18T06:57:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:34.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=123 2026-02-18T06:57:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 123 = 123 2026-02-18T06:57:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-18T06:57:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:57:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_19': 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12607ee406df 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12607ee406df 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:57:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:48:59 2026 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:48:59 2026 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:48:59 2026 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b90f47da-e462-47c3-9ed6-43d166775473 2026-02-18T06:57:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.export 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.export 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.export 2026-02-18T06:57:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.export 2026-02-18T06:57:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.export 2026-02-18T06:57:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.export 2026-02-18T06:57:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_19.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_19.export 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_20 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:57:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:57:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:57:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:57:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:57:38.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:57:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:57:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:57:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-18T06:57:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T06:57:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:57:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:57:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:57:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:38.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:38.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-02-18T06:57:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T06:57:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:57:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:57:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-02-18T06:57:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-18T06:57:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_20 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:57:38.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:57:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:38.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_20 2026-02-18T06:57:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:57:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:57:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:57:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:57:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:57:38.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:57:38.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:57:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2026-02-18T06:57:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2026-02-18T06:57:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 125 2026-02-18T06:57:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_20 2026-02-18T06:57:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_20.status 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_20.status 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.status 2026-02-18T06:57:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_20.status 2026-02-18T06:57:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_20.status 2026-02-18T06:57:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.status 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.status 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2026-02-18T06:57:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:39.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:57:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=125 2026-02-18T06:57:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:57:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-18T06:57:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=55 2026-02-18T06:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 125 = 55 2026-02-18T06:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:57:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-18T06:57:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=55 2026-02-18T06:57:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 125 = 55 2026-02-18T06:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:57:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-18T06:57:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:40.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 125 = '' 2026-02-18T06:57:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:40.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:57:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.status 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-18T06:57:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:42.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:42.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=125 2026-02-18T06:57:42.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 125 = 125 2026-02-18T06:57:42.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:42.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:57:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_20': 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126edc34eb6f 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126edc34eb6f 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:57:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:07 2026 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:07 2026 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:07 2026 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 484fe09b-1bcd-49b2-9d6f-6019db73c59b 2026-02-18T06:57:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:57:43.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.export 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.export 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.export 2026-02-18T06:57:43.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.export 2026-02-18T06:57:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.export 2026-02-18T06:57:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:43.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.export 2026-02-18T06:57:43.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_20.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_20.export 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_21 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:43.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:57:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-18T06:57:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:44.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-18T06:57:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:45.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:57:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:57:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:57:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:57:45.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:45.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:45.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:45.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-18T06:57:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-18T06:57:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2026-02-18T06:57:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:57:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:57:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2026-02-18T06:57:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:57:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:57:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:57:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:57:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2026-02-18T06:57:46.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 127 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_21 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_21.status 2026-02-18T06:57:46.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_21.status 2026-02-18T06:57:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:46.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.status 2026-02-18T06:57:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_21.status 2026-02-18T06:57:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_21.status 2026-02-18T06:57:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.status 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.status 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2026-02-18T06:57:46.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:46.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:57:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=127 2026-02-18T06:57:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-18T06:57:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=56 2026-02-18T06:57:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 127 = 56 2026-02-18T06:57:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:57:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-18T06:57:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=56 2026-02-18T06:57:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 127 = 56 2026-02-18T06:57:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:57:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:47.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-18T06:57:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:47.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:47.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 127 = '' 2026-02-18T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:57:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.status 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-18T06:57:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=127 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 127 = 127 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:57:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:50.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-18T06:57:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_21': 2026-02-18T06:57:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1191cf603ebf 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1191cf603ebf 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:22 2026 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:22 2026 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:22 2026 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3ecdab12-dc8c-4836-8efc-36c5c66c919a 2026-02-18T06:57:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.export 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.export 2026-02-18T06:57:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.export 2026-02-18T06:57:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.export 2026-02-18T06:57:50.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.export 2026-02-18T06:57:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.export 2026-02-18T06:57:50.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_21.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_21.export 2026-02-18T06:57:50.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_22 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:57:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-18T06:57:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-18T06:57:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:57:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:57:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:57:52.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:57:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-18T06:57:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:57:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:57:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:57:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:57:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2026-02-18T06:57:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 129 2026-02-18T06:57:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_22 2026-02-18T06:57:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:57:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:57:53.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:57:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_22.status 2026-02-18T06:57:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_22.status 2026-02-18T06:57:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.status 2026-02-18T06:57:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_22.status 2026-02-18T06:57:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_22.status 2026-02-18T06:57:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.status 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.status 2026-02-18T06:57:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2026-02-18T06:57:53.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:53.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:57:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=129 2026-02-18T06:57:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:57:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-18T06:57:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=57 2026-02-18T06:57:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 129 = 57 2026-02-18T06:57:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-18T06:57:53.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=57 2026-02-18T06:57:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 129 = 57 2026-02-18T06:57:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:57:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-18T06:57:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:57:54.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:54.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:57:54.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:57:54.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:57:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:57:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 129 = '' 2026-02-18T06:57:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:57:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.status 2026-02-18T06:57:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-18T06:57:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:57:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:57:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=129 2026-02-18T06:57:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 129 = 129 2026-02-18T06:57:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:57:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:57.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-18T06:57:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_22': 2026-02-18T06:57:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:57:57.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5b9547cb6 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5b9547cb6 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:34 2026 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:34 2026 2026-02-18T06:57:57.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:34 2026 2026-02-18T06:57:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:57:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:57:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:57:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: caed0cd9-e185-4f04-9bf1-ef7abd391a34 2026-02-18T06:57:57.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:57:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-18T06:57:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T06:57:57.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:57:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.export 2026-02-18T06:57:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.export 2026-02-18T06:57:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.export 2026-02-18T06:57:57.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.export 2026-02-18T06:57:57.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:57.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.export 2026-02-18T06:57:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:57:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.export 2026-02-18T06:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_22.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_22.export 2026-02-18T06:57:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_23 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:57:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:57:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-18T06:57:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:57.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:57:58.656 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.454862784196036) > weight (0.25) 2026-02-18T06:57:58.656 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T06:57:58.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T06:57:58.657 INFO:tasks.rbd_mirror_thrash:waiting for 3.520934812484451 secs before reviving daemons 2026-02-18T06:57:58.657 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T06:57:58.656+0000 7f836088f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 48597) UID: 1000 2026-02-18T06:57:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:57:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:57:58.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:58.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:57:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:57:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:57:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:58.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:57:58.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:57:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:57:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:58.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:57:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:57:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:57:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:57:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:57:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:57:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:57:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:57:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:57:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:57:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:57:59.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:57:59.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:59.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:57:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:57:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:57:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:57:59.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:57:59.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:57:59.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:57:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:57:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:57:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:57:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:57:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:57:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:57:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 1 2026-02-18T06:58:00.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:00.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:00.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:00.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:00.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:00.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:00.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:00.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:00.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T06:58:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 2 2026-02-18T06:58:02.179 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T06:58:02.179 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:58:02.179 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T06:58:02.179 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T06:58:02.180 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T06:58:02.182 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T06:58:02.182 INFO:tasks.rbd_mirror_thrash:waiting for 81.1371667934026 secs before thrashing 2026-02-18T06:58:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:02.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:02.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:02.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:02.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:02.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:02.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 4 2026-02-18T06:58:06.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:06.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:06.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:06.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:06.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:06.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 8 2026-02-18T06:58:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:14.871 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:14.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 8 2026-02-18T06:58:17.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8494458262962475) > weight (0.25) 2026-02-18T06:58:17.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9800133212561125) > weight (0.25) 2026-02-18T06:58:17.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7491035751927659) > weight (0.25) 2026-02-18T06:58:17.991 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4234361354838375) > weight (0.25) 2026-02-18T06:58:17.991 INFO:tasks.rbd_mirror_thrash:waiting for 92.77245681250861 secs before thrashing 2026-02-18T06:58:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:23.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:23.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:23.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:23.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:23.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:23.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:23.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:23.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 8 2026-02-18T06:58:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:31.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:31.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:31.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-18T06:58:31.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:31.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:31.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:31.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:31.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:31.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-18T06:58:31.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-18T06:58:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-18T06:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-18T06:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:31.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:31.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:31.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:31.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-18T06:58:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:31.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:31.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2026-02-18T06:58:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 131 2026-02-18T06:58:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_23 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_23.status 2026-02-18T06:58:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_23.status 2026-02-18T06:58:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:31.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.status 2026-02-18T06:58:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_23.status 2026-02-18T06:58:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_23.status 2026-02-18T06:58:31.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.status 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.status 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2026-02-18T06:58:31.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:31.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=131 2026-02-18T06:58:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-18T06:58:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=58 2026-02-18T06:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 131 = 58 2026-02-18T06:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:31.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:32.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-18T06:58:32.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:32.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:32.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:32.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 131 = '' 2026-02-18T06:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:58:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.status 2026-02-18T06:58:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-18T06:58:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=131 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 131 = 131 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:58:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:34.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_23': 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126063b0db66 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126063b0db66 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:58:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:41 2026 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:41 2026 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:41 2026 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3e439ed2-471c-4885-bdf3-4e6705e30c66 2026-02-18T06:58:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:58:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.export 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.export 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.export 2026-02-18T06:58:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.export 2026-02-18T06:58:34.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.export 2026-02-18T06:58:34.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.export 2026-02-18T06:58:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_23.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_23.export 2026-02-18T06:58:34.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:34.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_24 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:58:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:58:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:58:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:36.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-18T06:58:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2026-02-18T06:58:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2026-02-18T06:58:37.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 133 2026-02-18T06:58:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_24 2026-02-18T06:58:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_24.status 2026-02-18T06:58:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_24.status 2026-02-18T06:58:37.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.status 2026-02-18T06:58:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_24.status 2026-02-18T06:58:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_24.status 2026-02-18T06:58:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.status 2026-02-18T06:58:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.status 2026-02-18T06:58:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2026-02-18T06:58:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=133 2026-02-18T06:58:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:37.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:37.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:37.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:37.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-18T06:58:37.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:37.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=59 2026-02-18T06:58:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 133 = 59 2026-02-18T06:58:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:37.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-18T06:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:37.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:37.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=59 2026-02-18T06:58:37.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 133 = 59 2026-02-18T06:58:37.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:58:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-18T06:58:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 133 = '' 2026-02-18T06:58:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.status 2026-02-18T06:58:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-18T06:58:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:40.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=133 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 133 = 133 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:58:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_24': 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126ee5aba65 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126ee5aba65 2026-02-18T06:58:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:49:55 2026 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:49:55 2026 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:49:55 2026 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 3756c8e9-5361-42c1-94d5-bb07eb3a568d 2026-02-18T06:58:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.export 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.export 2026-02-18T06:58:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.export 2026-02-18T06:58:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.export 2026-02-18T06:58:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.export 2026-02-18T06:58:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.export 2026-02-18T06:58:41.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_24.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_24.export 2026-02-18T06:58:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:41.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_25 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:41.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:58:41.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:41.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:41.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-18T06:58:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:58:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:58:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:58:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:42.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:42.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:42.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:42.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:42.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-18T06:58:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T06:58:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:42.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:42.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:42.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:42.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2026-02-18T06:58:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T06:58:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:42.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_25 2026-02-18T06:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:58:42.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:58:42.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:58:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:58:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:58:42.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:58:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:58:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_25 2026-02-18T06:58:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-18T06:58:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:42.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_25 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:42.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:58:42.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:58:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:58:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_25 2026-02-18T06:58:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:43.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:43.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:43.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2026-02-18T06:58:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2026-02-18T06:58:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 135 2026-02-18T06:58:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_25 2026-02-18T06:58:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_25.status 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_25.status 2026-02-18T06:58:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.status 2026-02-18T06:58:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_25.status 2026-02-18T06:58:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_25.status 2026-02-18T06:58:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.status 2026-02-18T06:58:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.status 2026-02-18T06:58:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2026-02-18T06:58:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=135 2026-02-18T06:58:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-18T06:58:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 135 = '' 2026-02-18T06:58:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-18T06:58:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:44.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:44.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 135 = '' 2026-02-18T06:58:44.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:44.487 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.status 2026-02-18T06:58:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-18T06:58:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=135 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 135 = 135 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:45.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:58:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-18T06:58:46.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_25': 2026-02-18T06:58:46.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:58:46.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11911cf97588 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11911cf97588 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:04 2026 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:04 2026 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:04 2026 2026-02-18T06:58:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:58:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:58:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:58:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: b243d0f2-c374-431a-9dbd-c55ec3dfca56 2026-02-18T06:58:46.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:58:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-18T06:58:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T06:58:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:58:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.export 2026-02-18T06:58:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.export 2026-02-18T06:58:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.export 2026-02-18T06:58:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.export 2026-02-18T06:58:46.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.export 2026-02-18T06:58:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:46.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.export 2026-02-18T06:58:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_25.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_25.export 2026-02-18T06:58:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_26 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:58:46.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:46.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-18T06:58:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:58:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:58:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:47.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:47.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:47.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-18T06:58:47.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-18T06:58:47.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:58:47.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:58:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-02-18T06:58:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:58:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:47.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:47.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:58:47.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:58:47.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:58:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:58:47.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-02-18T06:58:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:47.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2026-02-18T06:58:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 137 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_26 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_26.status 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_26.status 2026-02-18T06:58:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.status 2026-02-18T06:58:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_26.status 2026-02-18T06:58:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_26.status 2026-02-18T06:58:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.status 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.status 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2026-02-18T06:58:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=137 2026-02-18T06:58:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-18T06:58:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:49.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 137 = '' 2026-02-18T06:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:49.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-18T06:58:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:49.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 137 = '' 2026-02-18T06:58:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:58:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.status 2026-02-18T06:58:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-18T06:58:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=137 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 137 = 137 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:58:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-18T06:58:51.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_26': 2026-02-18T06:58:51.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:58:51.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f53ef7564b 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f53ef7564b 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:15 2026 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:15 2026 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:15 2026 2026-02-18T06:58:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:58:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:58:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:58:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: bcbf3cf4-e8fd-4d05-8819-fa57991913f7 2026-02-18T06:58:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:58:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-18T06:58:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T06:58:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:58:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.export 2026-02-18T06:58:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.export 2026-02-18T06:58:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.export 2026-02-18T06:58:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.export 2026-02-18T06:58:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:51.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.export 2026-02-18T06:58:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.export 2026-02-18T06:58:51.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_26.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_26.export 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_27 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:58:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-18T06:58:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-18T06:58:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:58:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:58:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:58:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:58:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:52.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:52.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-18T06:58:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2026-02-18T06:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2026-02-18T06:58:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 139 2026-02-18T06:58:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_27 2026-02-18T06:58:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:53.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_27.status 2026-02-18T06:58:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_27.status 2026-02-18T06:58:53.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.status 2026-02-18T06:58:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_27.status 2026-02-18T06:58:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_27.status 2026-02-18T06:58:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.status 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.status 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2026-02-18T06:58:53.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:53.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=139 2026-02-18T06:58:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-18T06:58:54.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:54.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:54.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:54.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:54.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:54.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 139 = '' 2026-02-18T06:58:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:54.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:54.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:54.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:54.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-18T06:58:54.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:54.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:58:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:58:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:58:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:58:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 139 = '' 2026-02-18T06:58:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:54.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.status 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-18T06:58:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:55.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=139 2026-02-18T06:58:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 139 = 139 2026-02-18T06:58:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:58:55.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-18T06:58:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_27': 2026-02-18T06:58:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:58:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:58:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 12601661a9df 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.12601661a9df 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:21 2026 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:21 2026 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:21 2026 2026-02-18T06:58:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:58:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:58:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:58:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: f601ab61-b7bf-4d56-836f-81dda8d76269 2026-02-18T06:58:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:58:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.export 2026-02-18T06:58:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.export 2026-02-18T06:58:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.export 2026-02-18T06:58:56.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.export 2026-02-18T06:58:56.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.export 2026-02-18T06:58:56.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:58:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.export 2026-02-18T06:58:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_27.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_27.export 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_28 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:58:56.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:58:57.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:58:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:57.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:58:58.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:58:58.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:58:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:58:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:58:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:58:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:58:58.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:58:58.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:58:58.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:58.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:58.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:58:58.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-18T06:58:58.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:58:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:58:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:58:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:58:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:58:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:58:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2026-02-18T06:58:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2026-02-18T06:58:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 141 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_28 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_28.status 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_28.status 2026-02-18T06:58:59.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.status 2026-02-18T06:58:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_28.status 2026-02-18T06:58:59.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_28.status 2026-02-18T06:58:59.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.status 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.status 2026-02-18T06:58:59.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2026-02-18T06:58:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:58:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=141 2026-02-18T06:58:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:58:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-18T06:58:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:58:59.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:58:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=63 2026-02-18T06:58:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 141 = 63 2026-02-18T06:58:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:58:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:58:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:58:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:58:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:58:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:58:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:58:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-18T06:58:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 141 = '' 2026-02-18T06:59:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:00.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:59:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:59:00.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:59:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-18T06:59:00.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 141 = '' 2026-02-18T06:59:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:59:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:59:02.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.status 2026-02-18T06:59:02.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-18T06:59:02.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:02.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=141 2026-02-18T06:59:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 141 = 141 2026-02-18T06:59:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-18T06:59:03.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_28': 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 126ea2d5f663 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.126ea2d5f663 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:29 2026 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:29 2026 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:29 2026 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 87cd4a83-4564-4b99-b3f5-5b048052b5e2 2026-02-18T06:59:03.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:03.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.export 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.export 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.export 2026-02-18T06:59:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.export 2026-02-18T06:59:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:03.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.export 2026-02-18T06:59:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:03.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.export 2026-02-18T06:59:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_28.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_28.export 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_29 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:03.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-18T06:59:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:03.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:03.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:05.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:05.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:05.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-18T06:59:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-18T06:59:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-02-18T06:59:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:05.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:05.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-02-18T06:59:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:05.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2026-02-18T06:59:06.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 143 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_29 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_29.status 2026-02-18T06:59:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_29.status 2026-02-18T06:59:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.status 2026-02-18T06:59:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_29.status 2026-02-18T06:59:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_29.status 2026-02-18T06:59:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.status 2026-02-18T06:59:06.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.status 2026-02-18T06:59:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2026-02-18T06:59:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=143 2026-02-18T06:59:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:06.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-18T06:59:06.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:06.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=64 2026-02-18T06:59:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 143 = 64 2026-02-18T06:59:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:06.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-18T06:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=64 2026-02-18T06:59:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 143 = 64 2026-02-18T06:59:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-18T06:59:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 143 = '' 2026-02-18T06:59:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.status 2026-02-18T06:59:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-18T06:59:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:09.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:09.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=143 2026-02-18T06:59:09.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 143 = 143 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_29': 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 119129fe72b5 2026-02-18T06:59:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.119129fe72b5 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:35 2026 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:35 2026 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:35 2026 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d01291bd-af69-4270-aa28-f3563eed7534 2026-02-18T06:59:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.export 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.export 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.export 2026-02-18T06:59:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.export 2026-02-18T06:59:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.export 2026-02-18T06:59:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.export 2026-02-18T06:59:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_29.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_29.export 2026-02-18T06:59:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_30 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-18T06:59:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-18T06:59:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-18T06:59:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:12.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2026-02-18T06:59:13.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 145 2026-02-18T06:59:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_30 2026-02-18T06:59:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_30.status 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_30.status 2026-02-18T06:59:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.status 2026-02-18T06:59:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_30.status 2026-02-18T06:59:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_30.status 2026-02-18T06:59:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.status 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.status 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2026-02-18T06:59:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:13.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=145 2026-02-18T06:59:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:13.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-18T06:59:13.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=65 2026-02-18T06:59:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 145 = 65 2026-02-18T06:59:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-18T06:59:13.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:13.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=65 2026-02-18T06:59:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 145 = 65 2026-02-18T06:59:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-18T06:59:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 145 = '' 2026-02-18T06:59:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:14.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.status 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-18T06:59:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:16.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=145 2026-02-18T06:59:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 145 = 145 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-18T06:59:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_30': 2026-02-18T06:59:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 127882a7c74 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.127882a7c74 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:47 2026 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:47 2026 2026-02-18T06:59:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:47 2026 2026-02-18T06:59:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 8ba02ff9-8f1b-476e-a125-0499e1932593 2026-02-18T06:59:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-18T06:59:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.export 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.export 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.export 2026-02-18T06:59:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.export 2026-02-18T06:59:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.export 2026-02-18T06:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:17.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.export 2026-02-18T06:59:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_30.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_30.export 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_31 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-18T06:59:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-18T06:59:18.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:18.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-18T06:59:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T06:59:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-18T06:59:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:18.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-18T06:59:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T06:59:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:18.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:18.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:18.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:59:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:59:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:59:18.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2026-02-18T06:59:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-18T06:59:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_31 2026-02-18T06:59:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:59:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:59:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:59:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:59:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:59:18.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_31 2026-02-18T06:59:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:19.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2026-02-18T06:59:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 147 2026-02-18T06:59:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_31 2026-02-18T06:59:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_31.status 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_31.status 2026-02-18T06:59:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.status 2026-02-18T06:59:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_31.status 2026-02-18T06:59:19.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_31.status 2026-02-18T06:59:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.status 2026-02-18T06:59:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.status 2026-02-18T06:59:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2026-02-18T06:59:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=147 2026-02-18T06:59:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:20.043 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-18T06:59:20.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:20.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 147 = '' 2026-02-18T06:59:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-18T06:59:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:20.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:20.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 147 = '' 2026-02-18T06:59:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.status 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-18T06:59:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:21.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=147 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 147 = 147 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-18T06:59:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_31': 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 127871baa3d0 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.127871baa3d0 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:50:59 2026 2026-02-18T06:59:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:50:59 2026 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:50:59 2026 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 68ddba5c-0c32-4a47-9248-13cbd388326b 2026-02-18T06:59:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-18T06:59:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.export 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.export 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.export 2026-02-18T06:59:22.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.export 2026-02-18T06:59:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.export 2026-02-18T06:59:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.export 2026-02-18T06:59:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_31.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_31.export 2026-02-18T06:59:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_32 2026-02-18T06:59:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-18T06:59:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:23.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:23.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:23.321 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T06:59:23.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T06:59:23.321 INFO:tasks.rbd_mirror_thrash:waiting for 8.71692366053712 secs before reviving daemons 2026-02-18T06:59:23.322 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T06:59:23.321+0000 7f369d11e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 36863) UID: 1000 2026-02-18T06:59:23.331 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:59:23.329+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f530e4e0 2026-02-18T06:59:23.331 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:59:23.329+0000 7f8d89fae640 -1 reset not still connected to 0x55c7f7494820 2026-02-18T06:59:23.332 INFO:tasks.ceph.cluster2.osd.2.trial051.stderr:2026-02-18T06:59:23.330+0000 7f8d89fae640 -1 reset not still connected to 0x55c7fb180dd0 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-18T06:59:24.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T06:59:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-18T06:59:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-18T06:59:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2026-02-18T06:59:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:59:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:59:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:59:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:59:24.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:59:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2026-02-18T06:59:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2026-02-18T06:59:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 149 2026-02-18T06:59:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_32 2026-02-18T06:59:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_32.status 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_32.status 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.status 2026-02-18T06:59:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_32.status 2026-02-18T06:59:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_32.status 2026-02-18T06:59:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.status 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.status 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2026-02-18T06:59:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=149 2026-02-18T06:59:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:25.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-18T06:59:25.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:25.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=67 2026-02-18T06:59:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 149 = 67 2026-02-18T06:59:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-18T06:59:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=67 2026-02-18T06:59:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 149 = 67 2026-02-18T06:59:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:26.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-18T06:59:26.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 149 = '' 2026-02-18T06:59:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.status 2026-02-18T06:59:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-18T06:59:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=149 2026-02-18T06:59:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 149 = 149 2026-02-18T06:59:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:28.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_32': 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1278f053d277 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1278f053d277 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:10 2026 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:10 2026 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:10 2026 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 4cc8e195-bdc7-4ab2-b53d-fb5ccf229629 2026-02-18T06:59:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.export 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.export 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.export 2026-02-18T06:59:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.export 2026-02-18T06:59:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.export 2026-02-18T06:59:29.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:29.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.export 2026-02-18T06:59:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_32.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_32.export 2026-02-18T06:59:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_33 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:29.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:30.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:30.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:30.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-18T06:59:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T06:59:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:30.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-02-18T06:59:30.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T06:59:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T06:59:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T06:59:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T06:59:30.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T06:59:30.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:30.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:30.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T06:59:30.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2026-02-18T06:59:30.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-18T06:59:30.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T06:59:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T06:59:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T06:59:30.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:30.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:30.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T06:59:30.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2026-02-18T06:59:31.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:31.019 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:31.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:31.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:31.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2026-02-18T06:59:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 151 2026-02-18T06:59:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_33 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_33.status 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_33.status 2026-02-18T06:59:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.status 2026-02-18T06:59:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_33.status 2026-02-18T06:59:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_33.status 2026-02-18T06:59:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.status 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.status 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2026-02-18T06:59:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:31.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=151 2026-02-18T06:59:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-18T06:59:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:32.039 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T06:59:32.039 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T06:59:32.039 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T06:59:32.039 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T06:59:32.040 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T06:59:32.041 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T06:59:32.041 INFO:tasks.rbd_mirror_thrash:waiting for 95.95024732153601 secs before thrashing 2026-02-18T06:59:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=68 2026-02-18T06:59:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 151 = 68 2026-02-18T06:59:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-18T06:59:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:32.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 151 = '' 2026-02-18T06:59:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.status 2026-02-18T06:59:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-18T06:59:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:33.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=151 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 151 = 151 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_33': 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11911cffaca3 2026-02-18T06:59:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11911cffaca3 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:24 2026 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:24 2026 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:24 2026 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 72f9e03c-1614-48e6-b6e3-1f918706551e 2026-02-18T06:59:34.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.export 2026-02-18T06:59:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.export 2026-02-18T06:59:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.export 2026-02-18T06:59:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.export 2026-02-18T06:59:34.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.export 2026-02-18T06:59:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.export 2026-02-18T06:59:34.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_33.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_33.export 2026-02-18T06:59:34.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_34 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:34.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-18T06:59:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-18T06:59:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-18T06:59:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-18T06:59:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2026-02-18T06:59:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2026-02-18T06:59:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 153 2026-02-18T06:59:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_34 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_34.status 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_34.status 2026-02-18T06:59:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.status 2026-02-18T06:59:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_34.status 2026-02-18T06:59:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_34.status 2026-02-18T06:59:37.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:37.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.status 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.status 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2026-02-18T06:59:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:37.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=153 2026-02-18T06:59:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-18T06:59:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=69 2026-02-18T06:59:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 153 = 69 2026-02-18T06:59:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-18T06:59:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 153 = '' 2026-02-18T06:59:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-18T06:59:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 153 = '' 2026-02-18T06:59:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.status 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-18T06:59:40.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=153 2026-02-18T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 153 = 153 2026-02-18T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-18T06:59:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-18T06:59:41.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_34': 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5e75670f1 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5e75670f1 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:36 2026 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:36 2026 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:36 2026 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1c347546-ec8a-4a65-84e2-e2bd5d979b24 2026-02-18T06:59:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.export 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.export 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.export 2026-02-18T06:59:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.export 2026-02-18T06:59:41.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:41.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.export 2026-02-18T06:59:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.export 2026-02-18T06:59:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_34.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_34.export 2026-02-18T06:59:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_35 2026-02-18T06:59:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-18T06:59:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-18T06:59:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:44.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-18T06:59:44.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:44.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:44.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-18T06:59:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-18T06:59:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:44.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:44.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2026-02-18T06:59:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:44.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:44.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:44.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2026-02-18T06:59:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:44.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2026-02-18T06:59:45.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 155 2026-02-18T06:59:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_35 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_35.status 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_35.status 2026-02-18T06:59:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.status 2026-02-18T06:59:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_35.status 2026-02-18T06:59:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_35.status 2026-02-18T06:59:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.status 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.status 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2026-02-18T06:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:45.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=155 2026-02-18T06:59:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:45.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:45.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:45.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-18T06:59:45.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:45.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=70 2026-02-18T06:59:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 155 = 70 2026-02-18T06:59:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:45.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:45.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-18T06:59:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=70 2026-02-18T06:59:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 155 = 70 2026-02-18T06:59:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-18T06:59:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 155 = '' 2026-02-18T06:59:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T06:59:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.status 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-18T06:59:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:48.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:48.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=155 2026-02-18T06:59:48.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 155 = 155 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-18T06:59:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_35': 2026-02-18T06:59:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1260c2d24bbc 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1260c2d24bbc 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:42 2026 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:42 2026 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:42 2026 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 26bf0afd-33b6-476b-b97e-600b51d08d3b 2026-02-18T06:59:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.export 2026-02-18T06:59:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.export 2026-02-18T06:59:49.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.export 2026-02-18T06:59:49.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.export 2026-02-18T06:59:49.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.export 2026-02-18T06:59:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:49.639 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.export 2026-02-18T06:59:49.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_35.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_35.export 2026-02-18T06:59:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:49.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_36 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:49.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-18T06:59:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:50.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6210025103090373) > weight (0.25) 2026-02-18T06:59:50.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.49005183632334504) > weight (0.25) 2026-02-18T06:59:50.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.36350810390640664) > weight (0.25) 2026-02-18T06:59:50.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.710705116905126) > weight (0.25) 2026-02-18T06:59:50.765 INFO:tasks.rbd_mirror_thrash:waiting for 111.24173168425068 secs before thrashing 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-18T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:50.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:50.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:50.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:50.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-18T06:59:50.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-18T06:59:50.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:50.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-18T06:59:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:50.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:50.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:50.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:50.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-18T06:59:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:50.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:50.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:50.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2026-02-18T06:59:50.946 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2026-02-18T06:59:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 157 2026-02-18T06:59:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_36 2026-02-18T06:59:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_36.status 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_36.status 2026-02-18T06:59:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.status 2026-02-18T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_36.status 2026-02-18T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_36.status 2026-02-18T06:59:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.status 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.status 2026-02-18T06:59:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2026-02-18T06:59:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=157 2026-02-18T06:59:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:52.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-18T06:59:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 157 = '' 2026-02-18T06:59:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:52.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-18T06:59:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 157 = '' 2026-02-18T06:59:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.status 2026-02-18T06:59:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-18T06:59:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:53.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=157 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 157 = 157 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T06:59:53.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_36': 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1278745be757 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1278745be757 2026-02-18T06:59:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:51:53 2026 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:51:53 2026 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:51:53 2026 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T06:59:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 0c2518ac-f371-4e3c-855f-dcdd605f3efc 2026-02-18T06:59:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.export 2026-02-18T06:59:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.export 2026-02-18T06:59:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.export 2026-02-18T06:59:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.export 2026-02-18T06:59:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:54.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.export 2026-02-18T06:59:54.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T06:59:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.export 2026-02-18T06:59:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_36.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_36.export 2026-02-18T06:59:54.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:54.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T06:59:54.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_37 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T06:59:54.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T06:59:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T06:59:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T06:59:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-18T06:59:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T06:59:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:56.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T06:59:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T06:59:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T06:59:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T06:59:56.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-18T06:59:56.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-18T06:59:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T06:59:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T06:59:56.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T06:59:56.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:56.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T06:59:56.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T06:59:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T06:59:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T06:59:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2026-02-18T06:59:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T06:59:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T06:59:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T06:59:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T06:59:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T06:59:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T06:59:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2026-02-18T06:59:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2026-02-18T06:59:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 159 2026-02-18T06:59:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_37 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_37.status 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_37.status 2026-02-18T06:59:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.status 2026-02-18T06:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_37.status 2026-02-18T06:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_37.status 2026-02-18T06:59:57.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T06:59:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.status 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.status 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2026-02-18T06:59:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=159 2026-02-18T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:57.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-18T06:59:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=72 2026-02-18T06:59:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 159 = 72 2026-02-18T06:59:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:57.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-18T06:59:57.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:57.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=72 2026-02-18T06:59:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 159 = 72 2026-02-18T06:59:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:57.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T06:59:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T06:59:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T06:59:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T06:59:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T06:59:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-18T06:59:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T06:59:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T06:59:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T06:59:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T06:59:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T06:59:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T06:59:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T06:59:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 159 = '' 2026-02-18T06:59:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T06:59:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:00:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T07:00:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:00:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.status 2026-02-18T07:00:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-18T07:00:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=159 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 159 = 159 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-18T07:00:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_37': 2026-02-18T07:00:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1191dddd38b2 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1191dddd38b2 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:04 2026 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:04 2026 2026-02-18T07:00:01.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:04 2026 2026-02-18T07:00:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 2d943d33-932e-4e88-9180-1e3322e2eaf7 2026-02-18T07:00:01.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-18T07:00:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:01.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.export 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.export 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.export 2026-02-18T07:00:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.export 2026-02-18T07:00:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.export 2026-02-18T07:00:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.export 2026-02-18T07:00:01.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_37.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_37.export 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_38 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:01.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:01.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-18T07:00:02.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:02.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:02.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:02.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:02.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:02.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-18T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-18T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:02.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:02.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:02.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:02.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:02.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2026-02-18T07:00:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2026-02-18T07:00:03.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 161 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_38 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_38.status 2026-02-18T07:00:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_38.status 2026-02-18T07:00:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.status 2026-02-18T07:00:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_38.status 2026-02-18T07:00:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_38.status 2026-02-18T07:00:03.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.status 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.status 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2026-02-18T07:00:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=161 2026-02-18T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-18T07:00:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:04.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 161 = '' 2026-02-18T07:00:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-18T07:00:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:04.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:04.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:04.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 161 = '' 2026-02-18T07:00:04.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:04.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:05.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.status 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-18T07:00:05.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=161 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 161 = 161 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:05.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:06.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-18T07:00:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_38': 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f557f7c1a6 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f557f7c1a6 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:06.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:10 2026 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:10 2026 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:10 2026 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c3f5441a-66c5-4aac-8a5c-285f1d0190dd 2026-02-18T07:00:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.export 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.export 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.export 2026-02-18T07:00:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.export 2026-02-18T07:00:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.export 2026-02-18T07:00:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:06.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.export 2026-02-18T07:00:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_38.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_38.export 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_39 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-18T07:00:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-18T07:00:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-18T07:00:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:08.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-18T07:00:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-18T07:00:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2026-02-18T07:00:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 163 2026-02-18T07:00:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_39 2026-02-18T07:00:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_39.status 2026-02-18T07:00:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_39.status 2026-02-18T07:00:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.status 2026-02-18T07:00:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_39.status 2026-02-18T07:00:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_39.status 2026-02-18T07:00:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:09.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.status 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.status 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2026-02-18T07:00:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=163 2026-02-18T07:00:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:09.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:09.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-18T07:00:09.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=74 2026-02-18T07:00:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 163 = 74 2026-02-18T07:00:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:09.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-18T07:00:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=74 2026-02-18T07:00:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 163 = 74 2026-02-18T07:00:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:09.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-18T07:00:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 163 = '' 2026-02-18T07:00:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:00:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.status 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-18T07:00:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=163 2026-02-18T07:00:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 163 = 163 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:13.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-18T07:00:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_39': 2026-02-18T07:00:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1260558690f8 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1260558690f8 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:19 2026 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:19 2026 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:19 2026 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1e647b33-04b4-4af9-be40-448754ebc6b9 2026-02-18T07:00:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.export 2026-02-18T07:00:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.export 2026-02-18T07:00:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.export 2026-02-18T07:00:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.export 2026-02-18T07:00:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.export 2026-02-18T07:00:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:13.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.export 2026-02-18T07:00:13.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_39.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_39.export 2026-02-18T07:00:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_40 2026-02-18T07:00:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-18T07:00:13.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:13.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:13.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:13.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:13.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:13.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:13.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-18T07:00:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-18T07:00:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:14.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-18T07:00:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T07:00:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:14.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-02-18T07:00:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T07:00:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:00:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:00:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:00:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2026-02-18T07:00:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-18T07:00:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:14.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T07:00:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T07:00:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T07:00:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:14.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2026-02-18T07:00:15.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:15.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:15.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:15.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2026-02-18T07:00:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2026-02-18T07:00:15.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 165 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_40 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_40.status 2026-02-18T07:00:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_40.status 2026-02-18T07:00:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.status 2026-02-18T07:00:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_40.status 2026-02-18T07:00:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_40.status 2026-02-18T07:00:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.status 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.status 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2026-02-18T07:00:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=165 2026-02-18T07:00:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-18T07:00:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:16.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 165 = '' 2026-02-18T07:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:16.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-18T07:00:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 165 = '' 2026-02-18T07:00:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.status 2026-02-18T07:00:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-18T07:00:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=165 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 165 = 165 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:18.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_40': 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1278933a537d 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1278933a537d 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:25 2026 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:25 2026 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:25 2026 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d79ef20b-93e5-4c9a-898c-892bd3f13eb7 2026-02-18T07:00:18.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.export 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.export 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.export 2026-02-18T07:00:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.export 2026-02-18T07:00:18.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.export 2026-02-18T07:00:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.export 2026-02-18T07:00:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_40.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_40.export 2026-02-18T07:00:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_41 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:20.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-18T07:00:20.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:20.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-18T07:00:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T07:00:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-02-18T07:00:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:20.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-02-18T07:00:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T07:00:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:00:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:00:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:00:20.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-02-18T07:00:20.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-18T07:00:20.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:20.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_41 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:20.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T07:00:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T07:00:20.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:20.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:20.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:20.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_41 2026-02-18T07:00:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2026-02-18T07:00:21.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 167 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_41 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_41.status 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_41.status 2026-02-18T07:00:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.status 2026-02-18T07:00:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_41.status 2026-02-18T07:00:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_41.status 2026-02-18T07:00:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.status 2026-02-18T07:00:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.status 2026-02-18T07:00:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2026-02-18T07:00:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=167 2026-02-18T07:00:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-18T07:00:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=76 2026-02-18T07:00:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 167 = 76 2026-02-18T07:00:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-18T07:00:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:21.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=76 2026-02-18T07:00:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 167 = 76 2026-02-18T07:00:21.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:21.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-18T07:00:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:22.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 167 = '' 2026-02-18T07:00:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.status 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-18T07:00:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:24.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=167 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 167 = 167 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-18T07:00:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_41': 2026-02-18T07:00:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11918a58358c 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11918a58358c 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:39 2026 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:39 2026 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:39 2026 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: d085af16-259e-4393-9922-61c84825c9b6 2026-02-18T07:00:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.export 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.export 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.export 2026-02-18T07:00:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.export 2026-02-18T07:00:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.export 2026-02-18T07:00:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.export 2026-02-18T07:00:25.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_41.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_41.export 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_42 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:25.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:26.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:26.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-18T07:00:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-18T07:00:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:26.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:26.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:26.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:26.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:26.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:26.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:26.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2026-02-18T07:00:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:26.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:26.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:26.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:26.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2026-02-18T07:00:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 169 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_42 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_42.status 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_42.status 2026-02-18T07:00:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.status 2026-02-18T07:00:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_42.status 2026-02-18T07:00:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_42.status 2026-02-18T07:00:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.status 2026-02-18T07:00:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.status 2026-02-18T07:00:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2026-02-18T07:00:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:27.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=169 2026-02-18T07:00:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-18T07:00:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 169 = '' 2026-02-18T07:00:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:28.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-18T07:00:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:28.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 169 = '' 2026-02-18T07:00:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:28.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.status 2026-02-18T07:00:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-18T07:00:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:29.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=169 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 169 = 169 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:30.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-18T07:00:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_42': 2026-02-18T07:00:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f53facdd1d 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f53facdd1d 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:52:53 2026 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:52:53 2026 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:52:53 2026 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 532bd8d2-fa6c-4c79-9f24-faec877f5667 2026-02-18T07:00:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.export 2026-02-18T07:00:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.export 2026-02-18T07:00:30.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.export 2026-02-18T07:00:30.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.export 2026-02-18T07:00:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:30.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.export 2026-02-18T07:00:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.export 2026-02-18T07:00:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_42.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_42.export 2026-02-18T07:00:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_43 2026-02-18T07:00:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-18T07:00:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:30.741 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:32.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-18T07:00:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-18T07:00:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:32.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-18T07:00:32.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:32.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:32.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:32.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:32.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-18T07:00:32.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:32.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2026-02-18T07:00:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 171 2026-02-18T07:00:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_43 2026-02-18T07:00:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_43.status 2026-02-18T07:00:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_43.status 2026-02-18T07:00:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.status 2026-02-18T07:00:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_43.status 2026-02-18T07:00:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_43.status 2026-02-18T07:00:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.status 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.status 2026-02-18T07:00:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2026-02-18T07:00:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:33.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:33.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=171 2026-02-18T07:00:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:33.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-18T07:00:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:33.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=78 2026-02-18T07:00:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 171 = 78 2026-02-18T07:00:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:33.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:33.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-18T07:00:33.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=78 2026-02-18T07:00:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 171 = 78 2026-02-18T07:00:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-18T07:00:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:34.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:34.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:34.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:34.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:34.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:34.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 171 = '' 2026-02-18T07:00:34.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:34.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.status 2026-02-18T07:00:36.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-18T07:00:36.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=171 2026-02-18T07:00:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 171 = 171 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:37.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_43': 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1260b2b2fc1c 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1260b2b2fc1c 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:06 2026 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:06 2026 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:06 2026 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9142b2d3-2190-43ee-9ae4-a23d350d7595 2026-02-18T07:00:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.export 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.export 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.export 2026-02-18T07:00:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.export 2026-02-18T07:00:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:37.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.export 2026-02-18T07:00:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.export 2026-02-18T07:00:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_43.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_43.export 2026-02-18T07:00:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_44 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:37.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:38.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-18T07:00:38.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2026-02-18T07:00:39.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 173 2026-02-18T07:00:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_44 2026-02-18T07:00:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_44.status 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_44.status 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.status 2026-02-18T07:00:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_44.status 2026-02-18T07:00:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_44.status 2026-02-18T07:00:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:39.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.status 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.status 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2026-02-18T07:00:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=173 2026-02-18T07:00:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-18T07:00:40.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 173 = '' 2026-02-18T07:00:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-18T07:00:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 173 = '' 2026-02-18T07:00:40.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.status 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-18T07:00:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:41.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=173 2026-02-18T07:00:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 173 = 173 2026-02-18T07:00:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-18T07:00:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-18T07:00:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_44': 2026-02-18T07:00:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1278974cdc85 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1278974cdc85 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:20 2026 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:20 2026 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:20 2026 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 711413a6-3d43-4895-ae70-a3f48d733603 2026-02-18T07:00:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.export 2026-02-18T07:00:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.export 2026-02-18T07:00:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.export 2026-02-18T07:00:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.export 2026-02-18T07:00:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.export 2026-02-18T07:00:42.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.export 2026-02-18T07:00:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_44.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_44.export 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_45 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:42.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-18T07:00:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:44.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:44.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-18T07:00:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-18T07:00:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:44.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2026-02-18T07:00:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2026-02-18T07:00:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-18T07:00:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:44.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:00:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:00:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2026-02-18T07:00:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-18T07:00:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_45 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T07:00:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T07:00:44.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T07:00:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T07:00:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_45 2026-02-18T07:00:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:44.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:44.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:44.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2026-02-18T07:00:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 175 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_45 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_45.status 2026-02-18T07:00:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_45.status 2026-02-18T07:00:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.status 2026-02-18T07:00:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_45.status 2026-02-18T07:00:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_45.status 2026-02-18T07:00:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.status 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.status 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2026-02-18T07:00:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=175 2026-02-18T07:00:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:45.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-18T07:00:45.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:45.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=80 2026-02-18T07:00:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 175 = 80 2026-02-18T07:00:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:45.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:45.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-18T07:00:45.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=80 2026-02-18T07:00:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 175 = 80 2026-02-18T07:00:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-18T07:00:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 175 = '' 2026-02-18T07:00:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:46.664 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:00:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.status 2026-02-18T07:00:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-18T07:00:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:48.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=175 2026-02-18T07:00:48.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 175 = 175 2026-02-18T07:00:48.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:48.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:49.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-18T07:00:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_45': 2026-02-18T07:00:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:49.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 119128a98293 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.119128a98293 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:27 2026 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:27 2026 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:27 2026 2026-02-18T07:00:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: f6f2ce66-9eb2-475c-ba7d-b2bc4c3e0d27 2026-02-18T07:00:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.export 2026-02-18T07:00:49.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.export 2026-02-18T07:00:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.export 2026-02-18T07:00:49.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.export 2026-02-18T07:00:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.export 2026-02-18T07:00:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.export 2026-02-18T07:00:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_45.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_45.export 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_46 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:49.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-18T07:00:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-18T07:00:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:50.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:50.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:50.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:50.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:50.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:50.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:50.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-18T07:00:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T07:00:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:50.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:50.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:50.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:50.882 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:50.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-18T07:00:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T07:00:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:50.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:00:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:00:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:00:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:50.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-18T07:00:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-18T07:00:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T07:00:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T07:00:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T07:00:51.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:51.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:51.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:00:51.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-18T07:00:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:51.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:51.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:51.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:51.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2026-02-18T07:00:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 177 2026-02-18T07:00:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_46 2026-02-18T07:00:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_46.status 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_46.status 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.status 2026-02-18T07:00:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_46.status 2026-02-18T07:00:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_46.status 2026-02-18T07:00:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.status 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.status 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2026-02-18T07:00:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=177 2026-02-18T07:00:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-18T07:00:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:52.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 177 = '' 2026-02-18T07:00:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-18T07:00:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:52.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 177 = '' 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.status 2026-02-18T07:00:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-18T07:00:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=177 2026-02-18T07:00:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 177 = 177 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:00:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-18T07:00:54.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_46': 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f5f487919b 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f5f487919b 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:00:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:37 2026 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:37 2026 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:37 2026 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9587dfb7-d60a-4ccf-8fb2-510f8d803efa 2026-02-18T07:00:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.export 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.export 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.export 2026-02-18T07:00:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.export 2026-02-18T07:00:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.export 2026-02-18T07:00:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:00:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.export 2026-02-18T07:00:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_46.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_46.export 2026-02-18T07:00:54.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_47 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:00:54.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:00:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-18T07:00:55.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:55.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:00:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:00:56.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:00:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:00:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:00:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:00:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-18T07:00:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T07:00:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:56.188 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:56.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:00:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:00:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:00:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-18T07:00:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-18T07:00:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:00:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:00:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:00:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2026-02-18T07:00:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:00:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:00:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:00:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:00:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:00:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:00:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:00:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:00:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:00:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:56.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:00:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2026-02-18T07:00:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:00:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:00:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:00:56.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:00:56.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:00:56.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:00:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:00:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2026-02-18T07:00:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2026-02-18T07:00:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 179 2026-02-18T07:00:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_47 2026-02-18T07:00:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:00:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_47.status 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_47.status 2026-02-18T07:00:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.status 2026-02-18T07:00:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_47.status 2026-02-18T07:00:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_47.status 2026-02-18T07:00:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:00:57.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.status 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.status 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2026-02-18T07:00:57.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:57.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:00:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=179 2026-02-18T07:00:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:00:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:57.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-18T07:00:57.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=82 2026-02-18T07:00:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 179 = 82 2026-02-18T07:00:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:57.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:00:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-18T07:00:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=82 2026-02-18T07:00:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 179 = 82 2026-02-18T07:00:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:00:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-18T07:00:58.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:00:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:00:58.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:00:58.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:00:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 179 = '' 2026-02-18T07:00:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:00:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.status 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-18T07:01:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=179 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 179 = 179 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:01:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_47': 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1281b0ecebb 2026-02-18T07:01:01.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1281b0ecebb 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:44 2026 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:44 2026 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:44 2026 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 8a85931f-5e1b-4cb0-8882-4ef6fe35af93 2026-02-18T07:01:01.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:01.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:01:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.export 2026-02-18T07:01:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.export 2026-02-18T07:01:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.export 2026-02-18T07:01:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.export 2026-02-18T07:01:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.export 2026-02-18T07:01:01.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.export 2026-02-18T07:01:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_47.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_47.export 2026-02-18T07:01:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_48 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:01:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-18T07:01:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-18T07:01:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:01:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:01:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:01:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:01:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:01:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:01:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:01:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:02.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-18T07:01:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:01:02.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:01:02.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:01:02.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:01:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:01:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:01:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:01:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2026-02-18T07:01:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 181 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_48 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_48.status 2026-02-18T07:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_48.status 2026-02-18T07:01:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:03.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.status 2026-02-18T07:01:03.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_48.status 2026-02-18T07:01:03.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_48.status 2026-02-18T07:01:03.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.status 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.status 2026-02-18T07:01:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2026-02-18T07:01:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:01:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=181 2026-02-18T07:01:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:03.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-18T07:01:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:04.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:04.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:01:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:01:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:01:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:01:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 181 = '' 2026-02-18T07:01:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:04.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-18T07:01:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:04.512 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:04.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:01:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:01:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:01:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:01:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 181 = '' 2026-02-18T07:01:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:04.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.status 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-18T07:01:05.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:05.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=181 2026-02-18T07:01:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 181 = 181 2026-02-18T07:01:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:05.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:01:05.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-18T07:01:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_48': 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 119191378929 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.119191378929 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:01:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:53:52 2026 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:53:52 2026 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:53:52 2026 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 0f8c4865-0f0d-4a82-ba86-812a3b418b6a 2026-02-18T07:01:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:01:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-18T07:01:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.export 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.export 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.export 2026-02-18T07:01:06.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.export 2026-02-18T07:01:06.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:06.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.export 2026-02-18T07:01:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.export 2026-02-18T07:01:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_48.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_48.export 2026-02-18T07:01:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_49 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:01:07.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:01:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:07.992 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T07:01:07.992 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T07:01:07.993 INFO:tasks.rbd_mirror_thrash:waiting for 1.5572489344854346 secs before reviving daemons 2026-02-18T07:01:07.993 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:01:07.991+0000 7f42214e6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 52332) UID: 1000 2026-02-18T07:01:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-18T07:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:01:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:01:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:01:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:01:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:01:08.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:08.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:08.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:08.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-18T07:01:08.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-18T07:01:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:08.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:01:08.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:01:08.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:01:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2026-02-18T07:01:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-18T07:01:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:01:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:01:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:08.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:01:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:01:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2026-02-18T07:01:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:01:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:01:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:01:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:01:08.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:01:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:01:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:01:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-18T07:01:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2026-02-18T07:01:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 183 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_49 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_49.status 2026-02-18T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_49.status 2026-02-18T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.status 2026-02-18T07:01:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_49.status 2026-02-18T07:01:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_49.status 2026-02-18T07:01:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.status 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.status 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2026-02-18T07:01:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:01:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=183 2026-02-18T07:01:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:01:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-18T07:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=84 2026-02-18T07:01:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 183 = 84 2026-02-18T07:01:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:09.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:01:09.551 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T07:01:09.551 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:01:09.552 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T07:01:09.552 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T07:01:09.552 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T07:01:09.555 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T07:01:09.555 INFO:tasks.rbd_mirror_thrash:waiting for 76.5273535199453 secs before thrashing 2026-02-18T07:01:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-18T07:01:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:09.833 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=84 2026-02-18T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 183 = 84 2026-02-18T07:01:09.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:01:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-18T07:01:10.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:01:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:01:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:01:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:01:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 183 = '' 2026-02-18T07:01:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1.6 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.status 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-18T07:01:12.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:12.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=183 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 183 = 183 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:01:12.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_49': 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f53c9c94b7 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f53c9c94b7 2026-02-18T07:01:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:54:01 2026 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:54:01 2026 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:54:01 2026 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 9f47bbaa-cf2e-4ab0-887a-190da2abbcb4 2026-02-18T07:01:13.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.export 2026-02-18T07:01:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.export 2026-02-18T07:01:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.export 2026-02-18T07:01:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.export 2026-02-18T07:01:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.export 2026-02-18T07:01:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.export 2026-02-18T07:01:13.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_49.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_49.export 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_50 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=started 2026-02-18T07:01:13.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:13.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:13.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:14.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=started 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_result 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=stopped 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-18T07:01:14.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_instance=cluster1 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -i state 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance=cluster1 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local loop_instance 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-18T07:01:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sleep 0 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ seq 0 3 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:0 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:0 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=0 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:0 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:0 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=0 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=0 2026-02-18T07:01:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=0 2026-02-18T07:01:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-18T07:01:14.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:14.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:14.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-18T07:01:14.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-18T07:01:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T07:01:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:01:14.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:1 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-18T07:01:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=1 2026-02-18T07:01:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=1 2026-02-18T07:01:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-18T07:01:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-18T07:01:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-18T07:01:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T07:01:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:01:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:2 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:2 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=2 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=2 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-18T07:01:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:2 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:2 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=2 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=2 2026-02-18T07:01:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=2 2026-02-18T07:01:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-18T07:01:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-18T07:01:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2026-02-18T07:01:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-18T07:01:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd mirror status : get status for rbd mirror 2026-02-18T07:01:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:admin_socket: invalid command 2026-02-18T07:01:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_50 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1:3 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ shift 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local val=cluster1:3 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster_var_name=cluster 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local instance_var_name=instance 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=3 2026-02-18T07:01:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval cluster=cluster1 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ eval instance=3 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local local_cluster=cluster1:3 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster=cluster1 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local val=cluster1:3 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local cluster_var_name=local_cluster 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ local instance_var_name=instance 2026-02-18T07:01:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ cluster=cluster1 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ instance=3 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval local_cluster=cluster1 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ local_cluster=cluster1 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ eval instance=3 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ instance=3 2026-02-18T07:01:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-18T07:01:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:01:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:01:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-18T07:01:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_50 2026-02-18T07:01:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ return 0 2026-02-18T07:01:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ status_result=' "state": "Replaying"' 2026-02-18T07:01:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo ' "state": "Replaying"' 2026-02-18T07:01:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ grep -i Replaying 2026-02-18T07:01:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: "state": "Replaying" 2026-02-18T07:01:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=started 2026-02-18T07:01:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test started = started 2026-02-18T07:01:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = journal ']' 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' snapshot = snapshot ']' 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2026-02-18T07:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Snapshot ID: 185 2026-02-18T07:01:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_50 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_cluster=cluster1 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_pool=mirror 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local remote_pool=mirror 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster2-mirror-image_50.status 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster2-mirror-image_50.status 2026-02-18T07:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local status_log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.status 2026-02-18T07:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ mkfname cluster1-mirror-image_50.status 2026-02-18T07:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ echo cluster1-mirror-image_50.status 2026-02-18T07:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-18T07:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local local_status_log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.status 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.status 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2026-02-18T07:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:15.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-18T07:01:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snapshot_id=185 2026-02-18T07:01:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.2 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-18T07:01:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:01:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:01:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:01:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:01:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 185 = '' 2026-02-18T07:01:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.4 2026-02-18T07:01:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-18T07:01:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:01:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:-:1.1: Document is empty 2026-02-18T07:01:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: 2026-02-18T07:01:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:^ 2026-02-18T07:01:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id= 2026-02-18T07:01:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 185 = '' 2026-02-18T07:01:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-18T07:01:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.8 2026-02-18T07:01:17.121 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T07:01:17.119+0000 7f841a83d640 -1 reset not still connected to 0x55ddbe0e1930 2026-02-18T07:01:17.121 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T07:01:17.119+0000 7f841a83d640 -1 reset not still connected to 0x55ddc02389c0 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local log=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.status 2026-02-18T07:01:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-18T07:01:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-18T07:01:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-18T07:01:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local primary_snapshot_id=185 2026-02-18T07:01:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 185 = 185 2026-02-18T07:01:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-18T07:01:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-18T07:01:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:01:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-18T07:01:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_50': 2026-02-18T07:01:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 128 MiB in 32 objects 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 2 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 11f537c7a207 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.11f537c7a207 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering, non-primary 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 06:54:08 2026 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 06:54:08 2026 2026-02-18T07:01:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 06:54:08 2026 2026-02-18T07:01:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: protected: False 2026-02-18T07:01:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:01:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:01:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: efa032aa-a7f7-4e9f-a922-c3940dfa3a3f 2026-02-18T07:01:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: false 2026-02-18T07:01:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-18T07:01:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap_name=snap 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local ret=0 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local rmt_export=/tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.export 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local loc_export=/tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.export 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.export 2026-02-18T07:01:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.export 2026-02-18T07:01:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.export 2026-02-18T07:01:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-18T07:01:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cmp /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.export 2026-02-18T07:01:19.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -f /tmp/tmp.y3X31uzVRt/cluster2-mirror-image_50.export /tmp/tmp.y3X31uzVRt/cluster1-mirror-image_50.export 2026-02-18T07:01:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: delete many images' 2026-02-18T07:01:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T07:01:19.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T07:01:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 07:01:19 TEST: delete many images 2026-02-18T07:01:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 07:01:19 TEST: delete many images 2026-02-18T07:01:19.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 50 2026-02-18T07:01:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_1 2026-02-18T07:01:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-18T07:01:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:01:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:19.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-18T07:01:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:01:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:01:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:01:19.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:01:19.327 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:01:19.325+0000 7f8360526640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=1f218227-0cf6-470c-8df3-a73ed823e755 2026-02-18T07:01:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_2 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-18T07:01:21.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:01:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:01:21.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:01:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:01:21.352 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:21.350+0000 7f1d0f468640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288143ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:21.352 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:21.350+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x558287c69900 [3/e6c1e58d-7558-4bf6-9061-94218ae350a1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_3 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:22.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:01:22.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:22.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-18T07:01:23.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:01:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:01:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:01:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:01:23.352 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:23.350+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915f01a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:23.352 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:23.351+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a44000 [3/ef7fee7b-f8b3-4ef1-ade2-f49cb926b8d3] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:24.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_4 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:24.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-18T07:01:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:25.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-18T07:01:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:01:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:01:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:01:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:01:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:01:25.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:25.355+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d25fa76c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:25.357 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:25.355+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27e95400 [3/e7796cb1-112a-4c40-81fa-5cc2bae63d79] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:26.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_5 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:01:26.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:26.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-18T07:01:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:01:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:01:27.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:01:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:01:27.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:27.358+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27200ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:27.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:27.358+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27802c80 [3/db0ba49e-c26a-4f8a-bc0c-a15701fe2493] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_6 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-18T07:01:28.970 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T07:01:28.969+0000 7f841a83d640 -1 reset not still connected to 0x55ddbc90b110 2026-02-18T07:01:28.970 INFO:tasks.ceph.cluster1.osd.0.trial006.stderr:2026-02-18T07:01:28.969+0000 7f841a83d640 -1 reset not still connected to 0x55ddbcff6270 2026-02-18T07:01:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:01:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:01:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:01:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:01:29.365 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:29.364+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb147b520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:29.365 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:29.364+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9f180 [3/42f0c001-dae4-46f5-8bba-6fe81293309d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_7 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:30.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-18T07:01:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-18T07:01:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:01:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:31.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:01:31.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:31.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:31.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:01:31.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:01:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:01:31.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:31.365+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915f1380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:31.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:31.365+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f892162500 [3/21e2d0de-af6e-45ad-b644-f4eee184726c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_8 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-18T07:01:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:33.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-18T07:01:33.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:33.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:33.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:01:33.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:33.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:01:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:01:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:01:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:01:33.353 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:01:33.353+0000 7f8360526640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=19f68516-8b2a-45d7-b3fb-465c882305ae 2026-02-18T07:01:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_9 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:34.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-18T07:01:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:01:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:01:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:01:35.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:01:35.127 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:35.126+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb1640340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:35.128 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:35.126+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9ea00 [3/40b4e69c-a609-443c-8267-78314c9f7989] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_10 2026-02-18T07:01:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-18T07:01:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:01:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-18T07:01:37.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-18T07:01:37.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:37.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:37.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:01:37.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:37.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:01:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:01:37.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:01:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:01:37.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:37.137+0000 7f1d0f468640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55828836d520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:37.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:37.137+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859e280 [3/c38dfa61-f880-425c-89ec-99dda99f916d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:37.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:37.938+0000 7f1d0f468640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5582881449c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:37.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:37.938+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x55828859e280 [3/c38dfa61-f880-425c-89ec-99dda99f916d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:38.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:38.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:38.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_11 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-18T07:01:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-18T07:01:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T07:01:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_11 2026-02-18T07:01:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T07:01:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-18T07:01:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-18T07:01:39.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:39.133+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915f0680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:39.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:39.133+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a44a00 [3/4a7b60e8-59db-4352-8976-1ffb0ad77ed8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:40.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_12 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:40.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-18T07:01:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-18T07:01:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T07:01:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_12 2026-02-18T07:01:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T07:01:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-18T07:01:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-18T07:01:41.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:41.535+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288143ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:41.537 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:01:41.535+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859e500 [3/e98f19fc-6436-4e93-bab9-e22e39c2f301] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:42.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.43932588578008447) > weight (0.25) 2026-02-18T07:01:42.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5107657493635634) > weight (0.25) 2026-02-18T07:01:42.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5580361068275203) > weight (0.25) 2026-02-18T07:01:42.007 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.40048565715162876) > weight (0.25) 2026-02-18T07:01:42.007 INFO:tasks.rbd_mirror_thrash:waiting for 104.77895688064591 secs before thrashing 2026-02-18T07:01:42.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_13 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-18T07:01:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-18T07:01:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T07:01:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_13 2026-02-18T07:01:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T07:01:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-18T07:01:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-18T07:01:43.189 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:43.188+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb171f1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:43.189 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:43.188+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9ef00 [3/ac0bdee8-83d2-49fa-8584-10a7ba48c5e6] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:44.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:44.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:44.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_14 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-18T07:01:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-18T07:01:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T07:01:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_14 2026-02-18T07:01:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T07:01:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-18T07:01:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-18T07:01:45.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:45.130+0000 7fa92a826640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb2480680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:45.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:45.130+0000 7fa931033640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9e780 [3/28c18533-c7df-40a3-9c10-0a9ba74d304a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:45.374 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:45.373+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb2480b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:45.374 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:45.373+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9e780 [3/28c18533-c7df-40a3-9c10-0a9ba74d304a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_15 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-18T07:01:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-18T07:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T07:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_15 2026-02-18T07:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T07:01:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-18T07:01:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-18T07:01:47.196 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:47.194+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb24804e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:47.196 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:47.194+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9e500 [3/35047dfb-3412-4601-ba30-18dbf9de2aaf] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_16 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T07:01:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-18T07:01:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-18T07:01:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T07:01:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_16 2026-02-18T07:01:49.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:49.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:49.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T07:01:49.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-18T07:01:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-18T07:01:49.200 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:49.198+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915f11e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:49.200 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:49.198+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f892162280 [3/c8eb0f42-d61b-4ba7-ad8e-859cc232365a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_17 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-18T07:01:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_17 2026-02-18T07:01:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T07:01:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-18T07:01:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-18T07:01:51.202 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:51.200+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d272011e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:51.202 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:51.200+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27803180 [3/b71799a9-93c2-42d8-b47e-5494b730ec39] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:52.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_18 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T07:01:52.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-18T07:01:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:53.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-18T07:01:53.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:53.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T07:01:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_18 2026-02-18T07:01:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T07:01:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-18T07:01:53.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-18T07:01:53.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:53.201+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb2480340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:53.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:01:53.201+0000 7fa931834640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9ec80 [3/58feaf8b-9b70-499d-a121-13c9d6bfc92a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:54.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:54.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_19 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-18T07:01:55.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_19 2026-02-18T07:01:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T07:01:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-18T07:01:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-18T07:01:55.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:55.132+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915f0000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:55.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:01:55.132+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a44c80 [3/b90f47da-e462-47c3-9ed6-43d166775473] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:56.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_20 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T07:01:56.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-18T07:01:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_20 2026-02-18T07:01:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T07:01:57.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-18T07:01:57.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-18T07:01:57.210 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:57.208+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d272004e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:57.210 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:57.208+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27299900 [3/484fe09b-1bcd-49b2-9d6f-6019db73c59b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:01:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_21 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:01:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-18T07:01:59.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:01:59.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:01:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_21 2026-02-18T07:01:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:01:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:01:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T07:01:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-18T07:01:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-18T07:01:59.215 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:59.213+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d272001a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:01:59.215 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:01:59.213+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27803900 [3/3ecdab12-dc8c-4836-8efc-36c5c66c919a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:00.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_22 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-18T07:02:01.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_22 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T07:02:01.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-18T07:02:01.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-18T07:02:01.218 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:01.216+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288146d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:01.218 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:01.216+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x55828859ea00 [3/caed0cd9-e185-4f04-9bf1-ef7abd391a34] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_23 2026-02-18T07:02:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-18T07:02:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T07:02:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-18T07:02:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_23 2026-02-18T07:02:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T07:02:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-18T07:02:03.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-18T07:02:03.220 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:03.218+0000 7ff201cf0640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8913629c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:03.220 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:03.218+0000 7ff2084fd640 -1 rbd::mirror::ImageReplayer: 0x55f891a45180 [3/3e439ed2-471c-4885-bdf3-4e6705e30c66] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:04.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_24 2026-02-18T07:02:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-18T07:02:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T07:02:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-18T07:02:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-18T07:02:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T07:02:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_24 2026-02-18T07:02:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T07:02:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-18T07:02:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-18T07:02:05.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:05.133+0000 7f1d0f468640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288071ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:05.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:05.133+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x558288a6e500 [3/3756c8e9-5361-42c1-94d5-bb07eb3a568d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:06.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_25 2026-02-18T07:02:06.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-18T07:02:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T07:02:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-18T07:02:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-18T07:02:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T07:02:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_25 2026-02-18T07:02:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T07:02:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-18T07:02:07.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-18T07:02:07.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:07.132+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27200000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:07.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:07.132+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27803b80 [3/b243d0f2-c374-431a-9dbd-c55ec3dfca56] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_26 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-18T07:02:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-18T07:02:09.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T07:02:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:09.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_26 2026-02-18T07:02:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T07:02:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-18T07:02:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-18T07:02:09.142 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:09.140+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288140d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:09.142 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:09.140+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859ec80 [3/bcbf3cf4-e8fd-4d05-8819-fa57991913f7] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_27 2026-02-18T07:02:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-18T07:02:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T07:02:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:10.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-18T07:02:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_27 2026-02-18T07:02:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T07:02:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-18T07:02:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-18T07:02:11.142 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:11.140+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f8915e4d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:11.142 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:11.140+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a45400 [3/f601ab61-b7bf-4d56-836f-81dda8d76269] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:12.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_28 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-18T07:02:13.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_28 2026-02-18T07:02:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T07:02:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-18T07:02:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-18T07:02:13.146 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:13.144+0000 7f1d0f468640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288a7a4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:13.146 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:13.144+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x558288a6e780 [3/87cd4a83-4564-4b99-b3f5-5b048052b5e2] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_29 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-18T07:02:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-18T07:02:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T07:02:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_29 2026-02-18T07:02:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T07:02:15.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-18T07:02:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-18T07:02:15.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:15.132+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27471ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:15.134 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:15.132+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27803400 [3/d01291bd-af69-4270-aa28-f3563eed7534] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:16.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:16.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_30 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:16.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-18T07:02:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-18T07:02:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T07:02:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_30 2026-02-18T07:02:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T07:02:17.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-18T07:02:17.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-18T07:02:17.148 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:17.147+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288a7aea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:17.148 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:17.147+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859f180 [3/8ba02ff9-8f1b-476e-a125-0499e1932593] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_31 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:18.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-18T07:02:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-18T07:02:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T07:02:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_31 2026-02-18T07:02:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T07:02:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-18T07:02:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-18T07:02:19.151 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:19.150+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f890df3860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:19.151 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:19.150+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a45680 [3/68ddba5c-0c32-4a47-9248-13cbd388326b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:20.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_32 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:20.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-18T07:02:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-18T07:02:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T07:02:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_32 2026-02-18T07:02:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T07:02:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-18T07:02:21.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-18T07:02:21.158 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:02:21.157+0000 7fa92b027640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb2481a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:21.158 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:02:21.157+0000 7fa931033640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9fb80 [3/4cc8e195-bdc7-4ab2-b53d-fb5ccf229629] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_33 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-18T07:02:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_33 2026-02-18T07:02:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T07:02:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-18T07:02:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-18T07:02:23.161 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:23.160+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d274721a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:23.161 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:23.160+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27e94500 [3/72f9e03c-1614-48e6-b6e3-1f918706551e] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_34 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T07:02:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-18T07:02:25.047 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_34 2026-02-18T07:02:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T07:02:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-18T07:02:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-18T07:02:25.137 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:25.136+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288141040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:25.137 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:25.136+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x55828859f680 [3/1c347546-ec8a-4a65-84e2-e2bd5d979b24] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:26.083 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-18T07:02:26.083 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-18T07:02:26.083 INFO:tasks.rbd_mirror_thrash:waiting for 1.6238351145007046 secs before reviving daemons 2026-02-18T07:02:26.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:02:26.083+0000 7fc93930e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 56269) UID: 1000 2026-02-18T07:02:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_35 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T07:02:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-18T07:02:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_35 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T07:02:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-18T07:02:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-18T07:02:27.167 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:27.166+0000 7ff201cf0640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f89139c9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:27.167 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:27.166+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a45900 [3/26bf0afd-33b6-476b-b97e-600b51d08d3b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:27.708 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-18T07:02:27.708 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:02:27.708 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-18T07:02:27.708 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-18T07:02:27.709 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-18T07:02:27.711 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-18T07:02:27.711 INFO:tasks.rbd_mirror_thrash:waiting for 75.56529259137956 secs before thrashing 2026-02-18T07:02:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_36 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-18T07:02:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-18T07:02:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T07:02:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_36 2026-02-18T07:02:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T07:02:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-18T07:02:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-18T07:02:29.173 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:02:29.172+0000 7fa92a826640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x555bb2481ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:29.173 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:02:29.172+0000 7fa931033640 -1 rbd::mirror::ImageReplayer: 0x555bb1e9f400 [3/0c2518ac-f371-4e3c-855f-dcdd605f3efc] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_37 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-18T07:02:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_37 2026-02-18T07:02:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:02:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-18T07:02:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-18T07:02:31.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:31.173+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27472000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:31.178 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:31.177+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d2746d520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:31.178 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:31.177+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27803680 [3/2d943d33-932e-4e88-9180-1e3322e2eaf7] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_38 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-18T07:02:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_38 2026-02-18T07:02:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:02:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-18T07:02:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-18T07:02:33.176 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:33.175+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288a7a680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:33.176 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:33.175+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859ef00 [3/c3f5441a-66c5-4aac-8a5c-285f1d0190dd] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:34.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_39 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:02:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-18T07:02:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_39 2026-02-18T07:02:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:02:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-18T07:02:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-18T07:02:35.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:35.134+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f890ec5860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:35.135 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:35.134+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f891a45b80 [3/1e647b33-04b4-4af9-be40-448754ebc6b9] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_40 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-18T07:02:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-18T07:02:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:02:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_40 2026-02-18T07:02:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:02:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-18T07:02:37.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-18T07:02:37.167 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:02:37.166+0000 7f8360d27640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=d79ef20b-93e5-4c9a-898c-892bd3f13eb7 2026-02-18T07:02:38.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_41 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-18T07:02:39.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-18T07:02:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:02:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_41 2026-02-18T07:02:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:02:39.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-18T07:02:39.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-18T07:02:39.173 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:02:39.172+0000 7f8360d27640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=d085af16-259e-4393-9922-61c84825c9b6 2026-02-18T07:02:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_42 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-18T07:02:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_42 2026-02-18T07:02:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:02:41.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-18T07:02:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-18T07:02:41.186 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:41.184+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5582882dbd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:41.186 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:41.184+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x55828859fb80 [3/532bd8d2-fa6c-4c79-9f24-faec877f5667] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_43 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:02:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-18T07:02:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-18T07:02:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:02:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_43 2026-02-18T07:02:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:02:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-18T07:02:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-18T07:02:43.190 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:43.188+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f891603d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:43.191 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:43.188+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f892162000 [3/9142b2d3-2190-43ee-9ae4-a23d350d7595] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_44 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-18T07:02:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:45.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_44 2026-02-18T07:02:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:02:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-18T07:02:45.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-18T07:02:45.125 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:02:45.124+0000 7f8360d27640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=711413a6-3d43-4895-ae70-a3f48d733603 2026-02-18T07:02:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_45 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-18T07:02:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-18T07:02:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:02:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_45 2026-02-18T07:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:02:47.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-18T07:02:47.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-18T07:02:47.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:47.201+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27463a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:47.204 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:47.201+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27e94f00 [3/f6f2ce66-9eb2-475c-ba7d-b2bc4c3e0d27] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_46 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:02:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-18T07:02:49.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_46 2026-02-18T07:02:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:02:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-18T07:02:49.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-18T07:02:49.206 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:49.204+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558287ef5040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:49.206 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:49.204+0000 7f1d15474640 -1 rbd::mirror::ImageReplayer: 0x558288a6e000 [3/9587dfb7-d60a-4ccf-8fb2-510f8d803efa] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:50.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_47 2026-02-18T07:02:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-18T07:02:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:02:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-18T07:02:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_47 2026-02-18T07:02:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:02:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-18T07:02:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-18T07:02:51.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:51.201+0000 7ff2024f1640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f891603a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:51.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:02:51.201+0000 7ff208cfe640 -1 rbd::mirror::ImageReplayer: 0x55f890026780 [3/8a85931f-5e1b-4cb0-8882-4ef6fe35af93] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_48 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-18T07:02:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_48 2026-02-18T07:02:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:02:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-18T07:02:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-18T07:02:53.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:02:53.389+0000 7f8360d27640 -1 rbd::mirror::ImageMap: 0x56127f7bfe60 handle_peer_ack_remove: failed to remove global_image_id=0f8c4865-0f0d-4a82-ba86-812a3b418b6a 2026-02-18T07:02:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_49 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-18T07:02:55.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:55.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-18T07:02:55.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:02:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:55.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_49 2026-02-18T07:02:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:02:55.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-18T07:02:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-18T07:02:55.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:55.138+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27461d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:55.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:02:55.138+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27e95180 [3/9f47bbaa-cf2e-4ab0-887a-190da2abbcb4] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_50 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:02:56.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local snap=snap 2026-02-18T07:02:56.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-18T07:02:57.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing snap: 100% complete...done. 2026-02-18T07:02:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-18T07:02:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:02:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:02:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:02:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_50 2026-02-18T07:02:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:02:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:02:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-18T07:02:57.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-18T07:02:57.216 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:57.214+0000 7f1d0ec67640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x558288a7bba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:02:57.216 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:02:57.214+0000 7f1d15c75640 -1 rbd::mirror::ImageReplayer: 0x55828859f900 [3/efa032aa-a7f7-4e9f-a922-c3940dfa3a3f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:02:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-18T07:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-18T07:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T07:02:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 07:02:58 TEST: image deletions should propagate 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 07:02:58 TEST: image deletions should propagate 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_count=0 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local count 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local last_count=0 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ true 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T07:02:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T07:02:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T07:02:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'images: ' 2026-02-18T07:02:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 2 2026-02-18T07:02:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T07:02:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ count=0 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 40 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in `seq 1 40` 2026-02-18T07:02:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 1 -ne 1 2026-02-18T07:02:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-18T07:02:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep 'image health:' 2026-02-18T07:02:58.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cut '-d ' -f 3 2026-02-18T07:02:58.237 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: failed to query services: (13) Permission denied 2026-02-18T07:02:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ state=OK 2026-02-18T07:02:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = ERROR 2026-02-18T07:02:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test OK = OK 2026-02-18T07:02:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.238 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 50 2026-02-18T07:02:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:02:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T07:02:58.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:58.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:58.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-18T07:02:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:02:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_2 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:02:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T07:02:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:02:58.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:02:58.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_3 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:02:58.744 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:02:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T07:02:58.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:58.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:58.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-18T07:02:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:02:58.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T07:02:58.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_4 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:02:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T07:02:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:02:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_5 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:02:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T07:02:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:59.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:02:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_6 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:02:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T07:02:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:02:59.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_7 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:02:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:02:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:02:59.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T07:02:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:02:59.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:02:59.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T07:02:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_8 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:02:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T07:02:59.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T07:02:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_9 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:02:59.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T07:02:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:02:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:02:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:02:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-18T07:02:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:02:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:02:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:02:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_10 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:02:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:02:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T07:03:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-18T07:03:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:03:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_11 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-18T07:03:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_11 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_11 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-18T07:03:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-18T07:03:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_11 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_11 2026-02-18T07:03:00.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_11 2026-02-18T07:03:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-18T07:03:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_12 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_12 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_12 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-18T07:03:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-18T07:03:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-18T07:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_12 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_12 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_12 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-18T07:03:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-18T07:03:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_13 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_13 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_13 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-18T07:03:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-18T07:03:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_13 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_13 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_13 2026-02-18T07:03:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-18T07:03:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-18T07:03:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_14 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_14 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_14 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-18T07:03:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-18T07:03:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_14 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_14 2026-02-18T07:03:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_14 2026-02-18T07:03:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-18T07:03:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-18T07:03:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_15 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_15 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_15 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-18T07:03:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-18T07:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-18T07:03:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_15 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_15 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_15 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-18T07:03:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-18T07:03:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_16 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_16 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_16 2026-02-18T07:03:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-18T07:03:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:00.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-18T07:03:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:00.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_16 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_16 2026-02-18T07:03:01.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_16 2026-02-18T07:03:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-18T07:03:01.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-18T07:03:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_17 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_17 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_17 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-18T07:03:01.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-18T07:03:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_17 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_17 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_17 2026-02-18T07:03:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-18T07:03:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-18T07:03:01.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_18 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_18 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_18 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-18T07:03:01.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-18T07:03:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_18 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_18 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_18 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-18T07:03:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-18T07:03:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_19 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T07:03:01.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_19 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_19 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-18T07:03:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-18T07:03:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-18T07:03:01.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_19 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_19 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_19 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-18T07:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_20 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_20 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_20 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-18T07:03:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-18T07:03:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_20 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_20 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_20 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-18T07:03:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-18T07:03:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_21 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_21 2026-02-18T07:03:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_21 2026-02-18T07:03:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-18T07:03:01.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.671 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-18T07:03:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_21 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_21 2026-02-18T07:03:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_21 2026-02-18T07:03:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-18T07:03:01.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-18T07:03:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_22 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_22 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_22 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-18T07:03:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-18T07:03:01.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-18T07:03:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_22 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_22 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_22 2026-02-18T07:03:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-18T07:03:01.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_23 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_23 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_23 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-18T07:03:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:01.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-18T07:03:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:01.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_23 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_23 2026-02-18T07:03:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_23 2026-02-18T07:03:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-18T07:03:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_24 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_24 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_24 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-18T07:03:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-18T07:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_24 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_24 2026-02-18T07:03:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_24 2026-02-18T07:03:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-18T07:03:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_25 2026-02-18T07:03:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_25 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_25 2026-02-18T07:03:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-18T07:03:02.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-18T07:03:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-18T07:03:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_25 2026-02-18T07:03:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_25 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_25 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-18T07:03:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-18T07:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_26 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_26 2026-02-18T07:03:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_26 2026-02-18T07:03:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-18T07:03:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-18T07:03:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-18T07:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_26 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_26 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_26 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-18T07:03:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_27 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_27 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_27 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-18T07:03:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-18T07:03:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_27 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_27 2026-02-18T07:03:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_27 2026-02-18T07:03:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-18T07:03:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-18T07:03:02.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_28 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_28 2026-02-18T07:03:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_28 2026-02-18T07:03:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-18T07:03:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-18T07:03:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-18T07:03:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_28 2026-02-18T07:03:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_28 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_28 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-18T07:03:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_29 2026-02-18T07:03:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_29 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_29 2026-02-18T07:03:02.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-18T07:03:02.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-18T07:03:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_29 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_29 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_29 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-18T07:03:02.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-18T07:03:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_30 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_30 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_30 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-18T07:03:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-18T07:03:03.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-18T07:03:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_30 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_30 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_30 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-18T07:03:03.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-18T07:03:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_31 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_31 2026-02-18T07:03:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_31 2026-02-18T07:03:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-18T07:03:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-18T07:03:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-18T07:03:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_31 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_31 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_31 2026-02-18T07:03:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-18T07:03:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-18T07:03:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.303 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_32 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_32 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_32 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-18T07:03:03.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-18T07:03:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-18T07:03:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_32 2026-02-18T07:03:03.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_32 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_32 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-18T07:03:03.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-18T07:03:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_33 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_33 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_33 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-18T07:03:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.476 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-18T07:03:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_33 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_33 2026-02-18T07:03:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_33 2026-02-18T07:03:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-18T07:03:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_34 2026-02-18T07:03:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_34 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_34 2026-02-18T07:03:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-18T07:03:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-18T07:03:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.626 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_34 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_34 2026-02-18T07:03:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_34 2026-02-18T07:03:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-18T07:03:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-18T07:03:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_35 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_35 2026-02-18T07:03:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_35 2026-02-18T07:03:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-18T07:03:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-18T07:03:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_35 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:03.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_35 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_35 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-18T07:03:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-18T07:03:03.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_36 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_36 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_36 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-18T07:03:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-18T07:03:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_36 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_36 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_36 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-18T07:03:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_37 2026-02-18T07:03:04.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_37 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_37 2026-02-18T07:03:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-18T07:03:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-18T07:03:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_37 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_37 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_37 2026-02-18T07:03:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-18T07:03:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_38 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-18T07:03:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_38 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_38 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-18T07:03:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-18T07:03:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_38 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_38 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_38 2026-02-18T07:03:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-18T07:03:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-18T07:03:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_39 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_39 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_39 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-18T07:03:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-18T07:03:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-18T07:03:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_39 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_39 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_39 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-18T07:03:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_40 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_40 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_40 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-18T07:03:04.510 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-18T07:03:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_40 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_40 2026-02-18T07:03:04.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_40 2026-02-18T07:03:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-18T07:03:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.657 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-18T07:03:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_41 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_41 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_41 2026-02-18T07:03:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-18T07:03:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-18T07:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_41 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_41 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_41 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-18T07:03:04.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_42 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_42 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_42 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-18T07:03:04.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-18T07:03:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_42 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:04.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_42 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_42 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-18T07:03:04.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-18T07:03:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_43 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_43 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_43 2026-02-18T07:03:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-18T07:03:04.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:04.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-18T07:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_43 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_43 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_43 2026-02-18T07:03:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-18T07:03:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_44 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_44 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_44 2026-02-18T07:03:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-18T07:03:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-18T07:03:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_44 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_44 2026-02-18T07:03:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_44 2026-02-18T07:03:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-18T07:03:05.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_45 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-18T07:03:05.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_45 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_45 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-18T07:03:05.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-18T07:03:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_45 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_45 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_45 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-18T07:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_46 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-18T07:03:05.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_46 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_46 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-18T07:03:05.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-18T07:03:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:05.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-18T07:03:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_46 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_46 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_46 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-18T07:03:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-18T07:03:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_47 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_47 2026-02-18T07:03:05.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_47 2026-02-18T07:03:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-18T07:03:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-18T07:03:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.693 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_47 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_47 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_47 2026-02-18T07:03:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-18T07:03:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_48 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_48 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_48 2026-02-18T07:03:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-18T07:03:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.740 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-18T07:03:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-18T07:03:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_48 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_48 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_48 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-18T07:03:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-18T07:03:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_49 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_49 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_49 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-18T07:03:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-18T07:03:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_49 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:06.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_49 2026-02-18T07:03:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_49 2026-02-18T07:03:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-18T07:03:06.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_50 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_50 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_50 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-18T07:03:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:06.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-18T07:03:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-18T07:03:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_50 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_50 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_50 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-18T07:03:06.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-18T07:03:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T07:03:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 07:03:06 TEST: delete images during bootstrap 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 07:03:06 TEST: delete images during bootstrap 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=image 2026-02-18T07:03:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-18T07:03:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd: mirroring is already configured for image mode 2026-02-18T07:03:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-18T07:03:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=image 2026-02-18T07:03:06.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-18T07:03:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd: mirroring is already configured for image mode 2026-02-18T07:03:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ start_mirror cluster1 2026-02-18T07:03:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance 2026-02-18T07:03:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local val=cluster1 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster_var_name=cluster 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local instance_var_name=instance 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cluster=cluster1 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=cluster1 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ instance=0 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval cluster=cluster1 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ cluster=cluster1 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ eval instance=0 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ instance=0 2026-02-18T07:03:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=test 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 1 10 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_1 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_1 2026-02-18T07:03:06.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-18T07:03:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:03:06.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-18T07:03:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:03:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-18T07:03:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_1': 2026-02-18T07:03:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:03:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1935dfba2977 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1935dfba2977 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:03:06 2026 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:03:06 2026 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:03:06 2026 2026-02-18T07:03:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:03:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:03:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: c1989db3-8f55-4dbd-861d-5ff12f68f40e 2026-02-18T07:03:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:03:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-18T07:03:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:03:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T07:03:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:03:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:03:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 36128768 bytes 2026-02-18T07:03:07.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 79821824 bytes 2026-02-18T07:03:07.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 124375040 bytes 2026-02-18T07:03:07.366 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 160476160 bytes 2026-02-18T07:03:07.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 238332928 bytes 2026-02-18T07:03:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 341563392 bytes 2026-02-18T07:03:07.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 401876992 bytes 2026-02-18T07:03:07.694 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 486112256 bytes 2026-02-18T07:03:07.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 528465920 bytes 2026-02-18T07:03:07.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 560447488 bytes 2026-02-18T07:03:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 577863680 bytes 2026-02-18T07:03:07.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 602757120 bytes 2026-02-18T07:03:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 644572160 bytes 2026-02-18T07:03:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 697106432 bytes 2026-02-18T07:03:08.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 705763328 bytes 2026-02-18T07:03:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 709582848 bytes 2026-02-18T07:03:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 733035520 bytes 2026-02-18T07:03:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 749692928 bytes 2026-02-18T07:03:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 864267264 bytes 2026-02-18T07:03:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 920668160 bytes 2026-02-18T07:03:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 974187520 bytes 2026-02-18T07:03:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1008098304 bytes 2026-02-18T07:03:08.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1052947456 bytes 2026-02-18T07:03:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1107397632 bytes 2026-02-18T07:03:08.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1124300800 bytes 2026-02-18T07:03:08.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1156206592 bytes 2026-02-18T07:03:08.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1188428800 bytes 2026-02-18T07:03:08.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1208098816 bytes 2026-02-18T07:03:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1210862592 bytes 2026-02-18T07:03:08.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1220852736 bytes 2026-02-18T07:03:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1252630528 bytes 2026-02-18T07:03:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1253263360 bytes 2026-02-18T07:03:09.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1297734656 bytes 2026-02-18T07:03:09.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1356130304 bytes 2026-02-18T07:03:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1399420928 bytes 2026-02-18T07:03:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1442507776 bytes 2026-02-18T07:03:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1556389888 bytes 2026-02-18T07:03:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1557005312 bytes 2026-02-18T07:03:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1578584064 bytes 2026-02-18T07:03:09.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1638085632 bytes 2026-02-18T07:03:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1705954304 bytes 2026-02-18T07:03:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1735292928 bytes 2026-02-18T07:03:09.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1750020096 bytes 2026-02-18T07:03:09.688 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1785480192 bytes 2026-02-18T07:03:09.756 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1855536128 bytes 2026-02-18T07:03:09.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1888340992 bytes 2026-02-18T07:03:09.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1927066624 bytes 2026-02-18T07:03:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1938835456 bytes 2026-02-18T07:03:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2029088768 bytes 2026-02-18T07:03:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2059227136 bytes 2026-02-18T07:03:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2111571968 bytes 2026-02-18T07:03:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2178202624 bytes 2026-02-18T07:03:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2244877312 bytes 2026-02-18T07:03:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2321500160 bytes 2026-02-18T07:03:10.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2330960896 bytes 2026-02-18T07:03:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2366065664 bytes 2026-02-18T07:03:10.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2366659584 bytes 2026-02-18T07:03:10.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2371164160 bytes 2026-02-18T07:03:10.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2432065536 bytes 2026-02-18T07:03:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2459265024 bytes 2026-02-18T07:03:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2479709184 bytes 2026-02-18T07:03:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2493940736 bytes 2026-02-18T07:03:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2567820288 bytes 2026-02-18T07:03:10.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2584038400 bytes 2026-02-18T07:03:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2632931328 bytes 2026-02-18T07:03:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2636243968 bytes 2026-02-18T07:03:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2676070400 bytes 2026-02-18T07:03:11.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2701700096 bytes 2026-02-18T07:03:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2755475456 bytes 2026-02-18T07:03:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2793584640 bytes 2026-02-18T07:03:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2840252416 bytes 2026-02-18T07:03:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2918288384 bytes 2026-02-18T07:03:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2990444544 bytes 2026-02-18T07:03:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2991059968 bytes 2026-02-18T07:03:11.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3011567616 bytes 2026-02-18T07:03:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3061223424 bytes 2026-02-18T07:03:11.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3082007552 bytes 2026-02-18T07:03:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3112273920 bytes 2026-02-18T07:03:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3141682176 bytes 2026-02-18T07:03:11.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3154908160 bytes 2026-02-18T07:03:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3245103104 bytes 2026-02-18T07:03:11.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3276016640 bytes 2026-02-18T07:03:11.887 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3293298688 bytes 2026-02-18T07:03:11.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3296116736 bytes 2026-02-18T07:03:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3364230144 bytes 2026-02-18T07:03:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3413767168 bytes 2026-02-18T07:03:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3451024384 bytes 2026-02-18T07:03:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3479552000 bytes 2026-02-18T07:03:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3610855424 bytes 2026-02-18T07:03:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3736977408 bytes 2026-02-18T07:03:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3778967552 bytes 2026-02-18T07:03:12.538 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3866279936 bytes 2026-02-18T07:03:12.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3946240000 bytes 2026-02-18T07:03:12.685 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3977958400 bytes 2026-02-18T07:03:12.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4027441152 bytes 2026-02-18T07:03:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4062542848 bytes 2026-02-18T07:03:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4088224768 bytes 2026-02-18T07:03:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4160329728 bytes 2026-02-18T07:03:12.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4175027200 bytes 2026-02-18T07:03:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4243281920 bytes 2026-02-18T07:03:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4269907968 bytes 2026-02-18T07:03:13.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4310601728 bytes 2026-02-18T07:03:13.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4345160704 bytes 2026-02-18T07:03:13.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4484891648 bytes 2026-02-18T07:03:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4501844992 bytes 2026-02-18T07:03:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4568691712 bytes 2026-02-18T07:03:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4648752128 bytes 2026-02-18T07:03:13.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4711702528 bytes 2026-02-18T07:03:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4723000320 bytes 2026-02-18T07:03:13.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4784237568 bytes 2026-02-18T07:03:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4831247360 bytes 2026-02-18T07:03:13.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4855374848 bytes 2026-02-18T07:03:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4917852160 bytes 2026-02-18T07:03:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5013541888 bytes 2026-02-18T07:03:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5074142208 bytes 2026-02-18T07:03:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5074756608 bytes 2026-02-18T07:03:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5103363072 bytes 2026-02-18T07:03:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5128090624 bytes 2026-02-18T07:03:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5156716544 bytes 2026-02-18T07:03:14.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5186420736 bytes 2026-02-18T07:03:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5233720320 bytes 2026-02-18T07:03:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5279100928 bytes 2026-02-18T07:03:14.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5337274368 bytes 2026-02-18T07:03:14.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5374108672 bytes 2026-02-18T07:03:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5408376832 bytes 2026-02-18T07:03:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5483792384 bytes 2026-02-18T07:03:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5528521728 bytes 2026-02-18T07:03:14.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5603438592 bytes 2026-02-18T07:03:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5645785088 bytes 2026-02-18T07:03:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5647857664 bytes 2026-02-18T07:03:15.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5666957312 bytes 2026-02-18T07:03:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5687958528 bytes 2026-02-18T07:03:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5696137216 bytes 2026-02-18T07:03:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5752009728 bytes 2026-02-18T07:03:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5853485056 bytes 2026-02-18T07:03:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5939173376 bytes 2026-02-18T07:03:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5944909824 bytes 2026-02-18T07:03:15.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5963983872 bytes 2026-02-18T07:03:15.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6026779648 bytes 2026-02-18T07:03:15.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6084390912 bytes 2026-02-18T07:03:15.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6119384064 bytes 2026-02-18T07:03:15.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6180319232 bytes 2026-02-18T07:03:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6217326592 bytes 2026-02-18T07:03:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6288129024 bytes 2026-02-18T07:03:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6323341312 bytes 2026-02-18T07:03:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6398510080 bytes 2026-02-18T07:03:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:03:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=1191aed3b340 2026-02-18T07:03:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:16.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_1 present 1191aed3b340 2026-02-18T07:03:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=1191aed3b340 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:03:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=1191aed3b340 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1191aed3b340 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 1191aed3b340 -o 1191aed3b340 = 1191aed3b340 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-18T07:03:17.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:03:17.070+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127eeafc00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=186 2026-02-18T07:03:17.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:03:17.070+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127eeafc00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=187, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=ba, last_copied_object_number=128, snap_seqs={186=18446744073709551614}] 2026-02-18T07:03:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:03:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-18T07:03:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:03:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:03:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T07:03:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:03:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:22.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_1 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_1 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-18T07:03:22.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:22.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-18T07:03:22.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:22.232 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-18T07:03:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-18T07:03:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:03:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:22.289 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:03:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:03:22.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:03:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:03:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:03:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:03:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:03:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:03:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:25.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:03:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:03:25.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:03:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:03:26.787 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T07:03:26.788 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T07:03:26.788 INFO:tasks.rbd_mirror_thrash:waiting for 5.631410899778183 secs before reviving daemons 2026-02-18T07:03:26.789 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:03:26.788+0000 7fa932035640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 43860) UID: 1000 2026-02-18T07:03:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:29.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:03:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:03:29.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:03:29.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:29.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:03:32.420 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T07:03:32.420 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:03:32.421 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T07:03:32.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T07:03:32.421 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T07:03:32.424 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T07:03:32.424 INFO:tasks.rbd_mirror_thrash:waiting for 95.52652003361402 secs before thrashing 2026-02-18T07:03:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:03:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:03:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:03:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:03:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:03:43.277 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6737285545933558) > weight (0.25) 2026-02-18T07:03:43.278 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.844916175312618) > weight (0.25) 2026-02-18T07:03:43.278 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T07:03:43.278 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T07:03:43.278 INFO:tasks.rbd_mirror_thrash:waiting for 2.8607116008916256 secs before reviving daemons 2026-02-18T07:03:43.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:03:43.277+0000 7fcdb351a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 43271) UID: 1000 2026-02-18T07:03:46.140 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T07:03:46.140 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:03:46.141 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T07:03:46.141 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T07:03:46.141 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T07:03:46.144 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T07:03:46.144 INFO:tasks.rbd_mirror_thrash:waiting for 80.47679713490227 secs before thrashing 2026-02-18T07:03:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_1 2026-02-18T07:03:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_1 2026-02-18T07:03:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-18T07:03:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-18T07:03:53.894 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_2 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:03:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_2 2026-02-18T07:03:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-18T07:03:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:53.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:03:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-18T07:03:54.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:03:54.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_2': 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 19779ad562f8 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.19779ad562f8 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:03:54.049 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:03:53 2026 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:03:53 2026 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:03:53 2026 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 8d26c0ad-aa3b-4917-8f59-b4436716e248 2026-02-18T07:03:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:03:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-18T07:03:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:03:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=5 2026-02-18T07:03:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:03:54.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:03:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 23822336 bytes 2026-02-18T07:03:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 39725056 bytes 2026-02-18T07:03:54.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 71014400 bytes 2026-02-18T07:03:54.270 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 90314752 bytes 2026-02-18T07:03:54.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 138370048 bytes 2026-02-18T07:03:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 181495808 bytes 2026-02-18T07:03:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 204932096 bytes 2026-02-18T07:03:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 252866560 bytes 2026-02-18T07:03:54.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 341716992 bytes 2026-02-18T07:03:54.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 379642880 bytes 2026-02-18T07:03:54.703 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 400596992 bytes 2026-02-18T07:03:54.776 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 469542912 bytes 2026-02-18T07:03:54.820 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 483357696 bytes 2026-02-18T07:03:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 519831552 bytes 2026-02-18T07:03:54.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 630117376 bytes 2026-02-18T07:03:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 693810176 bytes 2026-02-18T07:03:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 759040000 bytes 2026-02-18T07:03:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 770418688 bytes 2026-02-18T07:03:55.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 789824512 bytes 2026-02-18T07:03:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 832419840 bytes 2026-02-18T07:03:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 880250880 bytes 2026-02-18T07:03:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 882967552 bytes 2026-02-18T07:03:55.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 905068544 bytes 2026-02-18T07:03:55.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 938177536 bytes 2026-02-18T07:03:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1000517632 bytes 2026-02-18T07:03:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1061784576 bytes 2026-02-18T07:03:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1085969408 bytes 2026-02-18T07:03:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1119500288 bytes 2026-02-18T07:03:55.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1175871488 bytes 2026-02-18T07:03:55.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1210939392 bytes 2026-02-18T07:03:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1254673408 bytes 2026-02-18T07:03:55.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1267083264 bytes 2026-02-18T07:03:55.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1269084160 bytes 2026-02-18T07:03:55.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1273856000 bytes 2026-02-18T07:03:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1347912704 bytes 2026-02-18T07:03:56.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1363082240 bytes 2026-02-18T07:03:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1398336512 bytes 2026-02-18T07:03:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1448674304 bytes 2026-02-18T07:03:56.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1511626752 bytes 2026-02-18T07:03:56.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1565170688 bytes 2026-02-18T07:03:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1609689088 bytes 2026-02-18T07:03:56.508 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1645758464 bytes 2026-02-18T07:03:56.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1647179776 bytes 2026-02-18T07:03:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1703390208 bytes 2026-02-18T07:03:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1729821696 bytes 2026-02-18T07:03:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1787766784 bytes 2026-02-18T07:03:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1831859200 bytes 2026-02-18T07:03:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1897767936 bytes 2026-02-18T07:03:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1975286784 bytes 2026-02-18T07:03:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2007536640 bytes 2026-02-18T07:03:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2056737792 bytes 2026-02-18T07:03:57.145 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2089577472 bytes 2026-02-18T07:03:57.212 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2136130560 bytes 2026-02-18T07:03:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2166569984 bytes 2026-02-18T07:03:57.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2167180288 bytes 2026-02-18T07:03:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2238283776 bytes 2026-02-18T07:03:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2260277248 bytes 2026-02-18T07:03:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2287315968 bytes 2026-02-18T07:03:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2337707008 bytes 2026-02-18T07:03:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2377981952 bytes 2026-02-18T07:03:57.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2408916992 bytes 2026-02-18T07:03:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2451866624 bytes 2026-02-18T07:03:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2477854720 bytes 2026-02-18T07:03:57.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2498618368 bytes 2026-02-18T07:03:57.864 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2547374080 bytes 2026-02-18T07:03:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2588686336 bytes 2026-02-18T07:03:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2605583360 bytes 2026-02-18T07:03:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2635759616 bytes 2026-02-18T07:03:58.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2639412224 bytes 2026-02-18T07:03:58.140 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2708440064 bytes 2026-02-18T07:03:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2768389120 bytes 2026-02-18T07:03:58.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2848259072 bytes 2026-02-18T07:03:58.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2872224768 bytes 2026-02-18T07:03:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2875599872 bytes 2026-02-18T07:03:58.426 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2891344896 bytes 2026-02-18T07:03:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2932157440 bytes 2026-02-18T07:03:58.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2987417600 bytes 2026-02-18T07:03:58.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3063719936 bytes 2026-02-18T07:03:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3082324992 bytes 2026-02-18T07:03:58.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3112783872 bytes 2026-02-18T07:03:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3147842560 bytes 2026-02-18T07:03:58.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3218265088 bytes 2026-02-18T07:03:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3235992576 bytes 2026-02-18T07:03:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3242842112 bytes 2026-02-18T07:03:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3243421696 bytes 2026-02-18T07:03:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3274007552 bytes 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:03:59.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:03:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:03:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=1191e4185519 2026-02-18T07:03:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:03:59.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_2 present 1191e4185519 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:03:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=1191e4185519 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:03:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=1191e4185519 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1191e4185519 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 1191e4185519 -o 1191e4185519 = 1191e4185519 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:03:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-18T07:03:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:03:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:03:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:03:59.215 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-18T07:03:59.305 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:03:59.303+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fbc3c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2026-02-18T07:03:59.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:03:59.303+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fbc3c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2026-02-18T07:04:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:04:04.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:04:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T07:04:04.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:04:04.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:04:04.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:04:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_2 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_2 2026-02-18T07:04:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-18T07:04:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:04.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:04.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:04.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-18T07:04:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-18T07:04:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:04.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:04.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:04.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:04.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:04:04.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:04.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:04.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:04.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:04.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:04.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:04:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:05.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:05.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:05.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:05.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:04:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:04:11.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:11.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:11.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:11.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:11.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:11.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:11.937 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:04:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_2 2026-02-18T07:04:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_2 2026-02-18T07:04:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-18T07:04:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-18T07:04:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:04:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:04:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:04:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_3 2026-02-18T07:04:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-18T07:04:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:04:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_3 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:04:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-18T07:04:37.144 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:04:37.148 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-18T07:04:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_3': 2026-02-18T07:04:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:04:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:04:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:04:37.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 19ad760779ac 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.19ad760779ac 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:04:36 2026 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:04:36 2026 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:04:36 2026 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 336ffdba-c638-479d-b65c-bba266590d88 2026-02-18T07:04:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:04:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-18T07:04:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:04:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=8 2026-02-18T07:04:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:04:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:04:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 40239104 bytes 2026-02-18T07:04:37.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 92707840 bytes 2026-02-18T07:04:37.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 112106496 bytes 2026-02-18T07:04:37.432 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 168975360 bytes 2026-02-18T07:04:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 184640512 bytes 2026-02-18T07:04:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 217694208 bytes 2026-02-18T07:04:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 249827328 bytes 2026-02-18T07:04:37.614 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 275236864 bytes 2026-02-18T07:04:37.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 290716672 bytes 2026-02-18T07:04:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 349936640 bytes 2026-02-18T07:04:37.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 439643136 bytes 2026-02-18T07:04:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 440493056 bytes 2026-02-18T07:04:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 490502144 bytes 2026-02-18T07:04:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 521318400 bytes 2026-02-18T07:04:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 569880576 bytes 2026-02-18T07:04:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 601591808 bytes 2026-02-18T07:04:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 634211328 bytes 2026-02-18T07:04:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 644852736 bytes 2026-02-18T07:04:38.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 675381248 bytes 2026-02-18T07:04:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 687273984 bytes 2026-02-18T07:04:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 766892032 bytes 2026-02-18T07:04:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 842759168 bytes 2026-02-18T07:04:38.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 902437888 bytes 2026-02-18T07:04:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 974056448 bytes 2026-02-18T07:04:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1011014656 bytes 2026-02-18T07:04:38.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1068198912 bytes 2026-02-18T07:04:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1109760000 bytes 2026-02-18T07:04:38.823 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1123721216 bytes 2026-02-18T07:04:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1139064832 bytes 2026-02-18T07:04:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1168214016 bytes 2026-02-18T07:04:38.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1196426240 bytes 2026-02-18T07:04:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1215478784 bytes 2026-02-18T07:04:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1255553024 bytes 2026-02-18T07:04:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1310048256 bytes 2026-02-18T07:04:39.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1310659584 bytes 2026-02-18T07:04:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1326068736 bytes 2026-02-18T07:04:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1362019328 bytes 2026-02-18T07:04:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1376026624 bytes 2026-02-18T07:04:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1394127872 bytes 2026-02-18T07:04:39.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1406489600 bytes 2026-02-18T07:04:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1443438592 bytes 2026-02-18T07:04:39.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1488266240 bytes 2026-02-18T07:04:39.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1549726720 bytes 2026-02-18T07:04:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1576036352 bytes 2026-02-18T07:04:39.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1576760320 bytes 2026-02-18T07:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1665236992 bytes 2026-02-18T07:04:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1682269184 bytes 2026-02-18T07:04:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1768487936 bytes 2026-02-18T07:04:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1824696320 bytes 2026-02-18T07:04:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1861234688 bytes 2026-02-18T07:04:40.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1903081472 bytes 2026-02-18T07:04:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1971778560 bytes 2026-02-18T07:04:40.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1999195136 bytes 2026-02-18T07:04:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2021371904 bytes 2026-02-18T07:04:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2049001472 bytes 2026-02-18T07:04:40.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2093144064 bytes 2026-02-18T07:04:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2121503744 bytes 2026-02-18T07:04:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2150154240 bytes 2026-02-18T07:04:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2215938048 bytes 2026-02-18T07:04:40.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2268691456 bytes 2026-02-18T07:04:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2272885760 bytes 2026-02-18T07:04:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2345616384 bytes 2026-02-18T07:04:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2409025536 bytes 2026-02-18T07:04:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2430780416 bytes 2026-02-18T07:04:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2446976000 bytes 2026-02-18T07:04:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2491146240 bytes 2026-02-18T07:04:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2530782208 bytes 2026-02-18T07:04:41.284 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2582576128 bytes 2026-02-18T07:04:41.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2647652352 bytes 2026-02-18T07:04:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2708460544 bytes 2026-02-18T07:04:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2732422144 bytes 2026-02-18T07:04:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2733458432 bytes 2026-02-18T07:04:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2787625984 bytes 2026-02-18T07:04:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2832019456 bytes 2026-02-18T07:04:41.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2851759104 bytes 2026-02-18T07:04:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2922134528 bytes 2026-02-18T07:04:41.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2942564352 bytes 2026-02-18T07:04:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3009313792 bytes 2026-02-18T07:04:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3049658368 bytes 2026-02-18T07:04:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3089577984 bytes 2026-02-18T07:04:42.070 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3114415104 bytes 2026-02-18T07:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3115063296 bytes 2026-02-18T07:04:42.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3170905088 bytes 2026-02-18T07:04:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3176316928 bytes 2026-02-18T07:04:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3176956928 bytes 2026-02-18T07:04:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3276664832 bytes 2026-02-18T07:04:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3277289472 bytes 2026-02-18T07:04:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3353408512 bytes 2026-02-18T07:04:42.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3425347584 bytes 2026-02-18T07:04:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3473545216 bytes 2026-02-18T07:04:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3523356672 bytes 2026-02-18T07:04:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3566885888 bytes 2026-02-18T07:04:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3642532864 bytes 2026-02-18T07:04:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3665379328 bytes 2026-02-18T07:04:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3698357248 bytes 2026-02-18T07:04:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3709707264 bytes 2026-02-18T07:04:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3762611200 bytes 2026-02-18T07:04:43.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3831664640 bytes 2026-02-18T07:04:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3845792768 bytes 2026-02-18T07:04:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3865556992 bytes 2026-02-18T07:04:43.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3900138496 bytes 2026-02-18T07:04:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3948516352 bytes 2026-02-18T07:04:43.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4010962944 bytes 2026-02-18T07:04:43.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4074515456 bytes 2026-02-18T07:04:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4120064000 bytes 2026-02-18T07:04:43.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4191052800 bytes 2026-02-18T07:04:43.890 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4282869760 bytes 2026-02-18T07:04:43.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4332436480 bytes 2026-02-18T07:04:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4356967424 bytes 2026-02-18T07:04:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4426591232 bytes 2026-02-18T07:04:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4452939776 bytes 2026-02-18T07:04:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4557193216 bytes 2026-02-18T07:04:44.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4605718528 bytes 2026-02-18T07:04:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4643142656 bytes 2026-02-18T07:04:44.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4664189952 bytes 2026-02-18T07:04:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4687758336 bytes 2026-02-18T07:04:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4729538560 bytes 2026-02-18T07:04:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4762827776 bytes 2026-02-18T07:04:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4808543232 bytes 2026-02-18T07:04:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4811487232 bytes 2026-02-18T07:04:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4838070272 bytes 2026-02-18T07:04:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4858213376 bytes 2026-02-18T07:04:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4889221120 bytes 2026-02-18T07:04:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4934410240 bytes 2026-02-18T07:04:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4948367360 bytes 2026-02-18T07:04:45.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5021719552 bytes 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:45.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:04:45.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=119158543824 2026-02-18T07:04:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:04:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:04:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_3 present 119158543824 2026-02-18T07:04:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=119158543824 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:04:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:04:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=119158543824 2026-02-18T07:04:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 119158543824 2026-02-18T07:04:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 119158543824 -o 119158543824 = 119158543824 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-18T07:04:45.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:04:45.430+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f644800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2026-02-18T07:04:45.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:04:45.430+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f644800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2026-02-18T07:04:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:04:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:04:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T07:04:50.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:04:50.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:04:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:04:50.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-18T07:04:50.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:04:50.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:50.569 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_3 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_3 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-18T07:04:50.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:04:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:50.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-18T07:04:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-18T07:04:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:04:50.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:04:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:04:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:04:50.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:50.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:50.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:04:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:04:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:04:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:04:51.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:51.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:51.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:04:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:04:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:53.799 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:04:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:04:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:04:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:04:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:04:57.801 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:04:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:04:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:04:57.858 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:04:57.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:04:57.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:05:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:05:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:05:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:05:05.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:05:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:05:06.621 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.73823044306154) > weight (0.25) 2026-02-18T07:05:06.622 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T07:05:06.622 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T07:05:06.622 INFO:tasks.rbd_mirror_thrash:waiting for 4.5831063938717485 secs before reviving daemons 2026-02-18T07:05:06.623 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:05:06.622+0000 7f2affc6a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 49725) UID: 1000 2026-02-18T07:05:07.951 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.685030584481933) > weight (0.25) 2026-02-18T07:05:07.951 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8693548730152629) > weight (0.25) 2026-02-18T07:05:07.951 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-18T07:05:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-18T07:05:07.952 INFO:tasks.rbd_mirror_thrash:waiting for 8.38841271078833 secs before reviving daemons 2026-02-18T07:05:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:05:07.952+0000 7f1d16476640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 30117) UID: 1000 2026-02-18T07:05:11.206 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T07:05:11.206 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:05:11.207 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T07:05:11.207 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T07:05:11.207 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T07:05:11.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T07:05:11.210 INFO:tasks.rbd_mirror_thrash:waiting for 85.72160172135432 secs before thrashing 2026-02-18T07:05:16.342 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-18T07:05:16.342 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:05:16.342 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-18T07:05:16.342 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-18T07:05:16.343 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-18T07:05:16.345 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-18T07:05:16.345 INFO:tasks.rbd_mirror_thrash:waiting for 73.180579001257 secs before thrashing 2026-02-18T07:05:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_3 2026-02-18T07:05:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_3 2026-02-18T07:05:21.921 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-18T07:05:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-18T07:05:22.063 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:05:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:05:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:05:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_4 2026-02-18T07:05:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:05:22.067 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_4 2026-02-18T07:05:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-18T07:05:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:05:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-18T07:05:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:05:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_4': 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 19f376117e62 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.19f376117e62 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:05:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:05:22 2026 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:05:22 2026 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:05:22 2026 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 16e7c6a3-0798-4f21-933e-021cee9e3d5b 2026-02-18T07:05:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:05:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-18T07:05:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:05:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T07:05:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:05:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:05:23.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 78086144 bytes 2026-02-18T07:05:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 111740928 bytes 2026-02-18T07:05:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 152143872 bytes 2026-02-18T07:05:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 156455936 bytes 2026-02-18T07:05:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 229651456 bytes 2026-02-18T07:05:23.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 246521856 bytes 2026-02-18T07:05:23.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 282979328 bytes 2026-02-18T07:05:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 326787072 bytes 2026-02-18T07:05:23.912 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 348383232 bytes 2026-02-18T07:05:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 405398528 bytes 2026-02-18T07:05:24.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 428260352 bytes 2026-02-18T07:05:24.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 504378368 bytes 2026-02-18T07:05:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 570115072 bytes 2026-02-18T07:05:24.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 583447552 bytes 2026-02-18T07:05:24.278 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 629000192 bytes 2026-02-18T07:05:24.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 695285760 bytes 2026-02-18T07:05:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 720432128 bytes 2026-02-18T07:05:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 817724416 bytes 2026-02-18T07:05:24.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 869319680 bytes 2026-02-18T07:05:24.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 945981440 bytes 2026-02-18T07:05:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 1038402560 bytes 2026-02-18T07:05:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 1045568512 bytes 2026-02-18T07:05:24.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1103177728 bytes 2026-02-18T07:05:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1148043264 bytes 2026-02-18T07:05:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1187827712 bytes 2026-02-18T07:05:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1188457472 bytes 2026-02-18T07:05:25.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1303745536 bytes 2026-02-18T07:05:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1311493120 bytes 2026-02-18T07:05:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1324554240 bytes 2026-02-18T07:05:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1362458624 bytes 2026-02-18T07:05:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1430065152 bytes 2026-02-18T07:05:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1458915328 bytes 2026-02-18T07:05:25.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1476900864 bytes 2026-02-18T07:05:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1516844032 bytes 2026-02-18T07:05:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1577101312 bytes 2026-02-18T07:05:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1599956992 bytes 2026-02-18T07:05:25.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1617395712 bytes 2026-02-18T07:05:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1719267328 bytes 2026-02-18T07:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1805259776 bytes 2026-02-18T07:05:25.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1866318848 bytes 2026-02-18T07:05:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1873222656 bytes 2026-02-18T07:05:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1891817472 bytes 2026-02-18T07:05:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1962005504 bytes 2026-02-18T07:05:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1966586880 bytes 2026-02-18T07:05:26.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1967497216 bytes 2026-02-18T07:05:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1980643328 bytes 2026-02-18T07:05:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2003100672 bytes 2026-02-18T07:05:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2075158528 bytes 2026-02-18T07:05:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2119749632 bytes 2026-02-18T07:05:26.343 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2152431616 bytes 2026-02-18T07:05:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2174103552 bytes 2026-02-18T07:05:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2199908352 bytes 2026-02-18T07:05:26.505 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2244938752 bytes 2026-02-18T07:05:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2285176832 bytes 2026-02-18T07:05:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2358925312 bytes 2026-02-18T07:05:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2434115584 bytes 2026-02-18T07:05:26.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2455094272 bytes 2026-02-18T07:05:26.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2486296576 bytes 2026-02-18T07:05:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2538885120 bytes 2026-02-18T07:05:26.970 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2581011456 bytes 2026-02-18T07:05:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2640066560 bytes 2026-02-18T07:05:27.109 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2662253568 bytes 2026-02-18T07:05:27.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2746872832 bytes 2026-02-18T07:05:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2761102336 bytes 2026-02-18T07:05:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2862648320 bytes 2026-02-18T07:05:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2948264960 bytes 2026-02-18T07:05:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2948873216 bytes 2026-02-18T07:05:27.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 3020546048 bytes 2026-02-18T07:05:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 3077120000 bytes 2026-02-18T07:05:27.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3078502400 bytes 2026-02-18T07:05:27.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3189591040 bytes 2026-02-18T07:05:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3202888704 bytes 2026-02-18T07:05:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3232637952 bytes 2026-02-18T07:05:27.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3279280128 bytes 2026-02-18T07:05:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3333911552 bytes 2026-02-18T07:05:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3416753152 bytes 2026-02-18T07:05:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3444420608 bytes 2026-02-18T07:05:28.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3515826176 bytes 2026-02-18T07:05:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3575425024 bytes 2026-02-18T07:05:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3624254464 bytes 2026-02-18T07:05:28.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3658281984 bytes 2026-02-18T07:05:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3711198208 bytes 2026-02-18T07:05:28.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3730829312 bytes 2026-02-18T07:05:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3754382336 bytes 2026-02-18T07:05:28.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3834368000 bytes 2026-02-18T07:05:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3922836480 bytes 2026-02-18T07:05:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3942825984 bytes 2026-02-18T07:05:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3983006720 bytes 2026-02-18T07:05:28.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 4048931840 bytes 2026-02-18T07:05:28.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 4110186496 bytes 2026-02-18T07:05:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 4125786112 bytes 2026-02-18T07:05:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 4205829120 bytes 2026-02-18T07:05:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 4254587904 bytes 2026-02-18T07:05:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 4306256896 bytes 2026-02-18T07:05:29.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 4370246656 bytes 2026-02-18T07:05:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4399204352 bytes 2026-02-18T07:05:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4433447936 bytes 2026-02-18T07:05:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4475076608 bytes 2026-02-18T07:05:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4543425536 bytes 2026-02-18T07:05:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4586598400 bytes 2026-02-18T07:05:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4590845952 bytes 2026-02-18T07:05:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4678052864 bytes 2026-02-18T07:05:29.790 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4684284928 bytes 2026-02-18T07:05:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4713135104 bytes 2026-02-18T07:05:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4755281920 bytes 2026-02-18T07:05:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4798228480 bytes 2026-02-18T07:05:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4834530304 bytes 2026-02-18T07:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4862926848 bytes 2026-02-18T07:05:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4869401600 bytes 2026-02-18T07:05:30.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4903619584 bytes 2026-02-18T07:05:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4972288000 bytes 2026-02-18T07:05:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 5048846336 bytes 2026-02-18T07:05:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 5109506048 bytes 2026-02-18T07:05:30.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 5187243008 bytes 2026-02-18T07:05:30.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 5187865600 bytes 2026-02-18T07:05:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5257050112 bytes 2026-02-18T07:05:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5312353280 bytes 2026-02-18T07:05:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5362930688 bytes 2026-02-18T07:05:30.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5403461632 bytes 2026-02-18T07:05:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5437380608 bytes 2026-02-18T07:05:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5456153600 bytes 2026-02-18T07:05:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5485813760 bytes 2026-02-18T07:05:31.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5523256320 bytes 2026-02-18T07:05:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5562819584 bytes 2026-02-18T07:05:31.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5583152128 bytes 2026-02-18T07:05:31.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5620227072 bytes 2026-02-18T07:05:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5654404096 bytes 2026-02-18T07:05:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5673091072 bytes 2026-02-18T07:05:31.362 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5722415104 bytes 2026-02-18T07:05:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5765210112 bytes 2026-02-18T07:05:31.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5830642688 bytes 2026-02-18T07:05:31.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5878172672 bytes 2026-02-18T07:05:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5914518528 bytes 2026-02-18T07:05:31.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5946657792 bytes 2026-02-18T07:05:31.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5957261312 bytes 2026-02-18T07:05:31.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5976111104 bytes 2026-02-18T07:05:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6064114688 bytes 2026-02-18T07:05:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6069319680 bytes 2026-02-18T07:05:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6127492096 bytes 2026-02-18T07:05:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6153995264 bytes 2026-02-18T07:05:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 6189384704 bytes 2026-02-18T07:05:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 6210975744 bytes 2026-02-18T07:05:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6253516800 bytes 2026-02-18T07:05:32.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6315326464 bytes 2026-02-18T07:05:32.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6340491264 bytes 2026-02-18T07:05:32.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6376358912 bytes 2026-02-18T07:05:32.449 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6421993472 bytes 2026-02-18T07:05:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6459111424 bytes 2026-02-18T07:05:32.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6481257472 bytes 2026-02-18T07:05:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6551549952 bytes 2026-02-18T07:05:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6620935168 bytes 2026-02-18T07:05:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6656411648 bytes 2026-02-18T07:05:32.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6660552704 bytes 2026-02-18T07:05:32.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6699881472 bytes 2026-02-18T07:05:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6780706816 bytes 2026-02-18T07:05:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6832364544 bytes 2026-02-18T07:05:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6860243968 bytes 2026-02-18T07:05:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6873109504 bytes 2026-02-18T07:05:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6945189888 bytes 2026-02-18T07:05:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6968059904 bytes 2026-02-18T07:05:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 7031647232 bytes 2026-02-18T07:05:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 7099450368 bytes 2026-02-18T07:05:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 7153364992 bytes 2026-02-18T07:05:33.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 7217826816 bytes 2026-02-18T07:05:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 7257583616 bytes 2026-02-18T07:05:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 7326274560 bytes 2026-02-18T07:05:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7494917120 bytes 2026-02-18T07:05:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7559070720 bytes 2026-02-18T07:05:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7620685824 bytes 2026-02-18T07:05:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7644871680 bytes 2026-02-18T07:05:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7663556608 bytes 2026-02-18T07:05:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7677696000 bytes 2026-02-18T07:05:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7688979456 bytes 2026-02-18T07:05:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7729721344 bytes 2026-02-18T07:05:34.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7734088704 bytes 2026-02-18T07:05:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7802425344 bytes 2026-02-18T07:05:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7825635328 bytes 2026-02-18T07:05:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7933389824 bytes 2026-02-18T07:05:34.550 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7972916224 bytes 2026-02-18T07:05:34.596 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7996494848 bytes 2026-02-18T07:05:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 8049392640 bytes 2026-02-18T07:05:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 8113829888 bytes 2026-02-18T07:05:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 8215398400 bytes 2026-02-18T07:05:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 8268080128 bytes 2026-02-18T07:05:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8272532480 bytes 2026-02-18T07:05:35.021 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8347060224 bytes 2026-02-18T07:05:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8364771328 bytes 2026-02-18T07:05:35.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8406870016 bytes 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:05:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:05:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:05:35.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=1191843d3288 2026-02-18T07:05:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:05:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_4 present 1191843d3288 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:05:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=1191843d3288 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:05:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=1191843d3288 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1191843d3288 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 1191843d3288 -o 1191843d3288 = 1191843d3288 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-18T07:05:35.427 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:05:35.425+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fe23000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2026-02-18T07:05:35.427 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:05:35.425+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fe23000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2026-02-18T07:05:38.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:05:38.137+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27200000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:05:38.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:05:38.137+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27e94780 [3/16e7c6a3-0798-4f21-933e-021cee9e3d5b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:05:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:05:40.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-18T07:05:40.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:05:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:05:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:05:40.460 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T07:05:40.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:05:40.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:05:40.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:05:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_4 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:05:40.564 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:05:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_4 2026-02-18T07:05:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-18T07:05:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:05:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-18T07:05:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:05:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:05:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:05:40.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:05:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:40.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:40.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:05:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:05:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:41.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:41.679 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:05:41.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:05:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:05:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:05:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:05:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:43.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:43.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:05:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:05:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:05:47.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:05:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:05:55.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:05:55.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:05:55.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:05:55.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:05:55.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:05:55.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:05:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:05:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:05:55.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:05:55.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:05:55.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:06:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_4 2026-02-18T07:06:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_4 2026-02-18T07:06:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-18T07:06:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-18T07:06:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:06:12.064 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_5 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:06:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:06:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:06:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_5 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:06:12.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-18T07:06:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:06:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_5': 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1a292d98a623 2026-02-18T07:06:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1a292d98a623 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:06:12 2026 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:06:12 2026 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:06:12 2026 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 31714ec5-1f49-4107-b758-db98eebff97f 2026-02-18T07:06:12.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:06:12.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-18T07:06:12.218 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=7 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:06:12.219 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:06:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 75053056 bytes 2026-02-18T07:06:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 125993984 bytes 2026-02-18T07:06:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 153151488 bytes 2026-02-18T07:06:12.515 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 211990528 bytes 2026-02-18T07:06:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 286913536 bytes 2026-02-18T07:06:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 324237312 bytes 2026-02-18T07:06:12.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 430338048 bytes 2026-02-18T07:06:12.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 459874304 bytes 2026-02-18T07:06:12.849 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 504024064 bytes 2026-02-18T07:06:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 538564608 bytes 2026-02-18T07:06:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 550040576 bytes 2026-02-18T07:06:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 569340928 bytes 2026-02-18T07:06:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 646918144 bytes 2026-02-18T07:06:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 690112512 bytes 2026-02-18T07:06:13.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 734720000 bytes 2026-02-18T07:06:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 760747008 bytes 2026-02-18T07:06:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 802259968 bytes 2026-02-18T07:06:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 850807808 bytes 2026-02-18T07:06:13.419 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 875530240 bytes 2026-02-18T07:06:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 888816640 bytes 2026-02-18T07:06:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 920484864 bytes 2026-02-18T07:06:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 958020608 bytes 2026-02-18T07:06:13.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1061829632 bytes 2026-02-18T07:06:13.721 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1105968128 bytes 2026-02-18T07:06:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1163991040 bytes 2026-02-18T07:06:13.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1222825984 bytes 2026-02-18T07:06:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1231412224 bytes 2026-02-18T07:06:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1322132480 bytes 2026-02-18T07:06:14.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1338390528 bytes 2026-02-18T07:06:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1358224384 bytes 2026-02-18T07:06:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1370153984 bytes 2026-02-18T07:06:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1397635072 bytes 2026-02-18T07:06:14.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1418064896 bytes 2026-02-18T07:06:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1440266240 bytes 2026-02-18T07:06:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1468679168 bytes 2026-02-18T07:06:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1544932352 bytes 2026-02-18T07:06:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1555829760 bytes 2026-02-18T07:06:14.453 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1571939328 bytes 2026-02-18T07:06:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1590558720 bytes 2026-02-18T07:06:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1641920512 bytes 2026-02-18T07:06:14.609 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1670662144 bytes 2026-02-18T07:06:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1699786752 bytes 2026-02-18T07:06:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1748962304 bytes 2026-02-18T07:06:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1763909632 bytes 2026-02-18T07:06:14.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1827739648 bytes 2026-02-18T07:06:14.896 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1847728128 bytes 2026-02-18T07:06:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1911211008 bytes 2026-02-18T07:06:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1927698432 bytes 2026-02-18T07:06:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2043416576 bytes 2026-02-18T07:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2079766528 bytes 2026-02-18T07:06:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2108282880 bytes 2026-02-18T07:06:15.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2210809856 bytes 2026-02-18T07:06:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2224866304 bytes 2026-02-18T07:06:15.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2257240064 bytes 2026-02-18T07:06:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2294710272 bytes 2026-02-18T07:06:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2399481856 bytes 2026-02-18T07:06:15.659 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2460530688 bytes 2026-02-18T07:06:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2521408512 bytes 2026-02-18T07:06:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2632919040 bytes 2026-02-18T07:06:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2657039360 bytes 2026-02-18T07:06:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2713274368 bytes 2026-02-18T07:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2788945920 bytes 2026-02-18T07:06:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2809032704 bytes 2026-02-18T07:06:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2830619648 bytes 2026-02-18T07:06:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2867070976 bytes 2026-02-18T07:06:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2890589184 bytes 2026-02-18T07:06:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2898526208 bytes 2026-02-18T07:06:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2936884224 bytes 2026-02-18T07:06:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2960324608 bytes 2026-02-18T07:06:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 3004932096 bytes 2026-02-18T07:06:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 3057029120 bytes 2026-02-18T07:06:16.559 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3104987136 bytes 2026-02-18T07:06:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3131898880 bytes 2026-02-18T07:06:16.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3146508288 bytes 2026-02-18T07:06:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3220247552 bytes 2026-02-18T07:06:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3261142016 bytes 2026-02-18T07:06:16.861 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3307987968 bytes 2026-02-18T07:06:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3320376320 bytes 2026-02-18T07:06:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3320990720 bytes 2026-02-18T07:06:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3337119744 bytes 2026-02-18T07:06:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3354437632 bytes 2026-02-18T07:06:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3416318976 bytes 2026-02-18T07:06:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3445904384 bytes 2026-02-18T07:06:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3484947456 bytes 2026-02-18T07:06:17.235 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3485571072 bytes 2026-02-18T07:06:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3514914816 bytes 2026-02-18T07:06:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3586440192 bytes 2026-02-18T07:06:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3607946240 bytes 2026-02-18T07:06:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3720514560 bytes 2026-02-18T07:06:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3744584704 bytes 2026-02-18T07:06:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3775324160 bytes 2026-02-18T07:06:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3822092288 bytes 2026-02-18T07:06:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3903271936 bytes 2026-02-18T07:06:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3929355264 bytes 2026-02-18T07:06:17.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3997502464 bytes 2026-02-18T07:06:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4069438464 bytes 2026-02-18T07:06:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4148028416 bytes 2026-02-18T07:06:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4179142656 bytes 2026-02-18T07:06:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4192604160 bytes 2026-02-18T07:06:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4232284160 bytes 2026-02-18T07:06:18.309 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4322828288 bytes 2026-02-18T07:06:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4361492480 bytes 2026-02-18T07:06:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4450614272 bytes 2026-02-18T07:06:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4586790912 bytes 2026-02-18T07:06:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4654157824 bytes 2026-02-18T07:06:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4712521728 bytes 2026-02-18T07:06:18.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4726734848 bytes 2026-02-18T07:06:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4733143040 bytes 2026-02-18T07:06:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4783102976 bytes 2026-02-18T07:06:18.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4842982400 bytes 2026-02-18T07:06:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4904936448 bytes 2026-02-18T07:06:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4927836160 bytes 2026-02-18T07:06:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4962626560 bytes 2026-02-18T07:06:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4993227776 bytes 2026-02-18T07:06:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:06:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:06:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:06:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:06:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:06:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:06:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:06:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:06:19.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=119127c2e3cd 2026-02-18T07:06:19.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:06:19.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:06:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_5 present 119127c2e3cd 2026-02-18T07:06:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:06:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=119127c2e3cd 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:06:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=119127c2e3cd 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 119127c2e3cd 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 119127c2e3cd -o 119127c2e3cd = 119127c2e3cd 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:06:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:06:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-18T07:06:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-18T07:06:19.481 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:06:19.480+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f0fe000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=190 2026-02-18T07:06:19.481 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:06:19.480+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f0fe000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=191, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=be, last_copied_object_number=128, snap_seqs={190=18446744073709551614}] 2026-02-18T07:06:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:06:24.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-18T07:06:24.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:06:24.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:06:24.490 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:06:24.513 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T07:06:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:06:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:06:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:06:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_5 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:06:24.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:06:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_5 2026-02-18T07:06:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-18T07:06:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:06:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:24.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:24.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:06:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:06:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:06:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:06:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:24.731 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:06:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:06:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:06:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:06:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:06:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:27.826 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:06:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:06:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:06:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:06:29.527 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T07:06:29.528 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T07:06:29.528 INFO:tasks.rbd_mirror_thrash:waiting for 6.910626917201195 secs before reviving daemons 2026-02-18T07:06:29.528 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:06:29.526+0000 7fb6108e4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 62684) UID: 1000 2026-02-18T07:06:31.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:06:36.439 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T07:06:36.440 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:06:36.440 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T07:06:36.440 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T07:06:36.440 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T07:06:36.443 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T07:06:36.443 INFO:tasks.rbd_mirror_thrash:waiting for 79.68451929765399 secs before thrashing 2026-02-18T07:06:36.932 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6766907253042264) > weight (0.25) 2026-02-18T07:06:36.932 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5713737469425053) > weight (0.25) 2026-02-18T07:06:36.933 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9536379551122869) > weight (0.25) 2026-02-18T07:06:36.933 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8542132505464176) > weight (0.25) 2026-02-18T07:06:36.933 INFO:tasks.rbd_mirror_thrash:waiting for 100.88649637801163 secs before thrashing 2026-02-18T07:06:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:06:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:06:39.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:06:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:06:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:06:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_5 2026-02-18T07:06:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_5 2026-02-18T07:06:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-18T07:06:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-18T07:06:56.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:06:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_6 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:06:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_6 2026-02-18T07:06:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-18T07:06:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:06:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:06:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-18T07:06:57.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:06:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-18T07:06:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_6': 2026-02-18T07:06:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:06:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:06:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:06:57.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1a657f5114e6 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1a657f5114e6 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:06:56 2026 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:06:56 2026 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:06:56 2026 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:06:57.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: bc8d9140-6c2a-4dff-9541-1d215c1bd6df 2026-02-18T07:06:57.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:06:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-18T07:06:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:06:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:06:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:06:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:06:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T07:06:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:06:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:06:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 18423808 bytes 2026-02-18T07:06:57.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 124821504 bytes 2026-02-18T07:06:57.398 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 142962688 bytes 2026-02-18T07:06:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 182948864 bytes 2026-02-18T07:06:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 221532160 bytes 2026-02-18T07:06:57.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 243951616 bytes 2026-02-18T07:06:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 313250816 bytes 2026-02-18T07:06:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 350486528 bytes 2026-02-18T07:06:57.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 381176832 bytes 2026-02-18T07:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 485795840 bytes 2026-02-18T07:06:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 509201408 bytes 2026-02-18T07:06:57.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 540570624 bytes 2026-02-18T07:06:58.009 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 597120000 bytes 2026-02-18T07:06:58.059 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 627183616 bytes 2026-02-18T07:06:58.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 685143040 bytes 2026-02-18T07:06:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 721470464 bytes 2026-02-18T07:06:58.246 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 759962624 bytes 2026-02-18T07:06:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 812394496 bytes 2026-02-18T07:06:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 818627584 bytes 2026-02-18T07:06:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 819247104 bytes 2026-02-18T07:06:58.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 945223680 bytes 2026-02-18T07:06:58.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 991774720 bytes 2026-02-18T07:06:58.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 1008490496 bytes 2026-02-18T07:06:58.643 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1053449216 bytes 2026-02-18T07:06:58.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1083586560 bytes 2026-02-18T07:06:58.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1086230528 bytes 2026-02-18T07:06:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1158540288 bytes 2026-02-18T07:06:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1165740032 bytes 2026-02-18T07:06:58.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1224260608 bytes 2026-02-18T07:06:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1259800576 bytes 2026-02-18T07:06:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1284622336 bytes 2026-02-18T07:06:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1318395904 bytes 2026-02-18T07:06:59.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1354576896 bytes 2026-02-18T07:06:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1409603584 bytes 2026-02-18T07:06:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1460796416 bytes 2026-02-18T07:06:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1493444608 bytes 2026-02-18T07:06:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1509679104 bytes 2026-02-18T07:06:59.416 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1544428544 bytes 2026-02-18T07:06:59.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1579083776 bytes 2026-02-18T07:06:59.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1622690816 bytes 2026-02-18T07:06:59.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1685486592 bytes 2026-02-18T07:06:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1691582464 bytes 2026-02-18T07:06:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1712631808 bytes 2026-02-18T07:06:59.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1760150528 bytes 2026-02-18T07:06:59.810 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1783115776 bytes 2026-02-18T07:06:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1819438080 bytes 2026-02-18T07:06:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1869477888 bytes 2026-02-18T07:06:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1909243904 bytes 2026-02-18T07:07:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1947686912 bytes 2026-02-18T07:07:00.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2007811072 bytes 2026-02-18T07:07:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2008423424 bytes 2026-02-18T07:07:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2077966336 bytes 2026-02-18T07:07:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2172664832 bytes 2026-02-18T07:07:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2232669184 bytes 2026-02-18T07:07:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2263102464 bytes 2026-02-18T07:07:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2311195648 bytes 2026-02-18T07:07:00.560 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2349921280 bytes 2026-02-18T07:07:00.618 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2395390976 bytes 2026-02-18T07:07:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2423874560 bytes 2026-02-18T07:07:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2446126080 bytes 2026-02-18T07:07:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2490035200 bytes 2026-02-18T07:07:00.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2524347392 bytes 2026-02-18T07:07:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2524966912 bytes 2026-02-18T07:07:00.919 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2549990400 bytes 2026-02-18T07:07:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2637568000 bytes 2026-02-18T07:07:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2651495424 bytes 2026-02-18T07:07:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2685780992 bytes 2026-02-18T07:07:01.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2705821696 bytes 2026-02-18T07:07:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2719276032 bytes 2026-02-18T07:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2802016256 bytes 2026-02-18T07:07:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2874748928 bytes 2026-02-18T07:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2892357632 bytes 2026-02-18T07:07:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2900791296 bytes 2026-02-18T07:07:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2958275584 bytes 2026-02-18T07:07:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2991306752 bytes 2026-02-18T07:07:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3050144768 bytes 2026-02-18T07:07:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3118447616 bytes 2026-02-18T07:07:01.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3151437824 bytes 2026-02-18T07:07:01.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3195350016 bytes 2026-02-18T07:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3238394880 bytes 2026-02-18T07:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3269574656 bytes 2026-02-18T07:07:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3352971264 bytes 2026-02-18T07:07:02.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3384879104 bytes 2026-02-18T07:07:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3410014208 bytes 2026-02-18T07:07:02.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3445533696 bytes 2026-02-18T07:07:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3457386496 bytes 2026-02-18T07:07:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3540790272 bytes 2026-02-18T07:07:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3578765312 bytes 2026-02-18T07:07:02.625 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3619651584 bytes 2026-02-18T07:07:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3695823872 bytes 2026-02-18T07:07:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3766174720 bytes 2026-02-18T07:07:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3815178240 bytes 2026-02-18T07:07:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3874700288 bytes 2026-02-18T07:07:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3942567936 bytes 2026-02-18T07:07:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3976620032 bytes 2026-02-18T07:07:03.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 4023397376 bytes 2026-02-18T07:07:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 4085618688 bytes 2026-02-18T07:07:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 4096463872 bytes 2026-02-18T07:07:03.341 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4144004096 bytes 2026-02-18T07:07:03.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4171384832 bytes 2026-02-18T07:07:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4220800000 bytes 2026-02-18T07:07:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4276606976 bytes 2026-02-18T07:07:03.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4280143872 bytes 2026-02-18T07:07:03.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4321387520 bytes 2026-02-18T07:07:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4384926720 bytes 2026-02-18T07:07:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4401439744 bytes 2026-02-18T07:07:03.788 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4415461376 bytes 2026-02-18T07:07:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4423268352 bytes 2026-02-18T07:07:03.906 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4480278528 bytes 2026-02-18T07:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4582708224 bytes 2026-02-18T07:07:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4769451008 bytes 2026-02-18T07:07:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4801014784 bytes 2026-02-18T07:07:04.266 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4813095936 bytes 2026-02-18T07:07:04.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4924986368 bytes 2026-02-18T07:07:04.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4949399552 bytes 2026-02-18T07:07:04.497 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 5000400896 bytes 2026-02-18T07:07:04.539 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 5014082560 bytes 2026-02-18T07:07:04.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 5077499904 bytes 2026-02-18T07:07:04.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 5104092160 bytes 2026-02-18T07:07:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 5122312192 bytes 2026-02-18T07:07:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 5131865088 bytes 2026-02-18T07:07:04.822 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5192379392 bytes 2026-02-18T07:07:04.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5209168896 bytes 2026-02-18T07:07:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5271586816 bytes 2026-02-18T07:07:05.017 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5321455616 bytes 2026-02-18T07:07:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5429696512 bytes 2026-02-18T07:07:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5495068672 bytes 2026-02-18T07:07:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5571544064 bytes 2026-02-18T07:07:05.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5576933376 bytes 2026-02-18T07:07:05.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5622444032 bytes 2026-02-18T07:07:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5664211968 bytes 2026-02-18T07:07:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5742107648 bytes 2026-02-18T07:07:05.633 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5829554176 bytes 2026-02-18T07:07:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5875216384 bytes 2026-02-18T07:07:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5956703232 bytes 2026-02-18T07:07:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5994979328 bytes 2026-02-18T07:07:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 6029886464 bytes 2026-02-18T07:07:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 6061515776 bytes 2026-02-18T07:07:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 6083978240 bytes 2026-02-18T07:07:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 6148536320 bytes 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:07:06.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=11918a987cb7 2026-02-18T07:07:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:07:06.223 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:07:06.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_6 present 11918a987cb7 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=11918a987cb7 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:06.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:07:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:07:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=11918a987cb7 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 11918a987cb7 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 11918a987cb7 -o 11918a987cb7 = 11918a987cb7 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:06.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-18T07:07:07.157 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:07:07.156+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5612800c9800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2026-02-18T07:07:07.158 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:07:07.156+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x5612800c9800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=192, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2026-02-18T07:07:08.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:07:08.138+0000 7ff78f4c9640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27472000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:07:08.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:07:08.138+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27e95180 [3/bc8d9140-6c2a-4dff-9541-1d215c1bd6df] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:07:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:07:12.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:07:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:12.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T07:07:12.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:07:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:07:12.193 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:07:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-18T07:07:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_6 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_6 2026-02-18T07:07:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-18T07:07:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-18T07:07:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-18T07:07:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-18T07:07:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:07:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:12.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:07:12.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:07:12.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:07:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:07:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:07:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:07:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:07:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:07:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:15.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:07:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:07:15.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:07:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:07:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:19.534 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:07:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:07:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:07:19.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:19.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:07:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:07:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:07:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:07:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:07:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:07:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_6 2026-02-18T07:07:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_6 2026-02-18T07:07:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-18T07:07:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-18T07:07:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_7 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:07:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_7 2026-02-18T07:07:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-18T07:07:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:07:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-18T07:07:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:07:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-18T07:07:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_7': 2026-02-18T07:07:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:07:44.124 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1a9bf61aad08 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1a9bf61aad08 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:07:43 2026 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:07:43 2026 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:07:43 2026 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1c0a18f6-7933-43f0-bd78-f8c6baaeb40e 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:07:44.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-18T07:07:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:44.126 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:07:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T07:07:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:07:44.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:07:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 599040 bytes 2026-02-18T07:07:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 57712640 bytes 2026-02-18T07:07:44.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 82078720 bytes 2026-02-18T07:07:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 179062784 bytes 2026-02-18T07:07:44.428 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 198614016 bytes 2026-02-18T07:07:44.485 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 241466368 bytes 2026-02-18T07:07:44.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 291220480 bytes 2026-02-18T07:07:44.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 294761472 bytes 2026-02-18T07:07:44.619 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 306841600 bytes 2026-02-18T07:07:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 341521408 bytes 2026-02-18T07:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 395584512 bytes 2026-02-18T07:07:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 454296576 bytes 2026-02-18T07:07:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 456748032 bytes 2026-02-18T07:07:44.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 485412864 bytes 2026-02-18T07:07:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 524504064 bytes 2026-02-18T07:07:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 569062400 bytes 2026-02-18T07:07:45.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 588795904 bytes 2026-02-18T07:07:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 607776768 bytes 2026-02-18T07:07:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 608399360 bytes 2026-02-18T07:07:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 681243648 bytes 2026-02-18T07:07:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 727699456 bytes 2026-02-18T07:07:45.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 764924928 bytes 2026-02-18T07:07:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 787043328 bytes 2026-02-18T07:07:45.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 787650560 bytes 2026-02-18T07:07:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 809715712 bytes 2026-02-18T07:07:45.502 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 847846400 bytes 2026-02-18T07:07:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 911680512 bytes 2026-02-18T07:07:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 934806528 bytes 2026-02-18T07:07:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 966184960 bytes 2026-02-18T07:07:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 996714496 bytes 2026-02-18T07:07:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1028471808 bytes 2026-02-18T07:07:45.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1083773952 bytes 2026-02-18T07:07:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1123713024 bytes 2026-02-18T07:07:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1167037440 bytes 2026-02-18T07:07:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1230304256 bytes 2026-02-18T07:07:46.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1334981632 bytes 2026-02-18T07:07:46.231 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1389444096 bytes 2026-02-18T07:07:46.305 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1457227776 bytes 2026-02-18T07:07:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1518848000 bytes 2026-02-18T07:07:46.415 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1522009088 bytes 2026-02-18T07:07:46.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1566069760 bytes 2026-02-18T07:07:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1650030592 bytes 2026-02-18T07:07:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1709213696 bytes 2026-02-18T07:07:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1733541888 bytes 2026-02-18T07:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1770456064 bytes 2026-02-18T07:07:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1771044864 bytes 2026-02-18T07:07:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1851759616 bytes 2026-02-18T07:07:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1882163200 bytes 2026-02-18T07:07:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1939558400 bytes 2026-02-18T07:07:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1950400512 bytes 2026-02-18T07:07:47.074 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1988319232 bytes 2026-02-18T07:07:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2060993536 bytes 2026-02-18T07:07:47.217 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2102681600 bytes 2026-02-18T07:07:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2202075136 bytes 2026-02-18T07:07:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2241906688 bytes 2026-02-18T07:07:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2260625408 bytes 2026-02-18T07:07:47.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2294888448 bytes 2026-02-18T07:07:47.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2352219136 bytes 2026-02-18T07:07:47.576 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2377720832 bytes 2026-02-18T07:07:47.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2433498112 bytes 2026-02-18T07:07:47.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2487790592 bytes 2026-02-18T07:07:47.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2517382144 bytes 2026-02-18T07:07:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2519810048 bytes 2026-02-18T07:07:47.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2551442432 bytes 2026-02-18T07:07:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2552041472 bytes 2026-02-18T07:07:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2576303104 bytes 2026-02-18T07:07:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2623454208 bytes 2026-02-18T07:07:48.062 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2670898176 bytes 2026-02-18T07:07:48.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2792624128 bytes 2026-02-18T07:07:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2895859712 bytes 2026-02-18T07:07:48.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2911295488 bytes 2026-02-18T07:07:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2928818176 bytes 2026-02-18T07:07:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2963006464 bytes 2026-02-18T07:07:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2992632832 bytes 2026-02-18T07:07:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3083620352 bytes 2026-02-18T07:07:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3128694784 bytes 2026-02-18T07:07:48.656 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3129296896 bytes 2026-02-18T07:07:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3140521984 bytes 2026-02-18T07:07:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3233253376 bytes 2026-02-18T07:07:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3267904512 bytes 2026-02-18T07:07:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3311280128 bytes 2026-02-18T07:07:48.941 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3337240576 bytes 2026-02-18T07:07:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3406553088 bytes 2026-02-18T07:07:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3419534336 bytes 2026-02-18T07:07:49.123 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3468274688 bytes 2026-02-18T07:07:49.182 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3509545984 bytes 2026-02-18T07:07:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3529147392 bytes 2026-02-18T07:07:49.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3563717632 bytes 2026-02-18T07:07:49.327 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3584721920 bytes 2026-02-18T07:07:49.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3604347904 bytes 2026-02-18T07:07:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3649067008 bytes 2026-02-18T07:07:49.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3668197376 bytes 2026-02-18T07:07:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3679007744 bytes 2026-02-18T07:07:49.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3697033216 bytes 2026-02-18T07:07:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3709847552 bytes 2026-02-18T07:07:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3768387584 bytes 2026-02-18T07:07:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3795381248 bytes 2026-02-18T07:07:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3801531392 bytes 2026-02-18T07:07:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3852179456 bytes 2026-02-18T07:07:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3886561280 bytes 2026-02-18T07:07:49.927 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3908916224 bytes 2026-02-18T07:07:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3940083712 bytes 2026-02-18T07:07:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3971650560 bytes 2026-02-18T07:07:50.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 3991055360 bytes 2026-02-18T07:07:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4025836544 bytes 2026-02-18T07:07:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4101727232 bytes 2026-02-18T07:07:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4124010496 bytes 2026-02-18T07:07:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4177792000 bytes 2026-02-18T07:07:50.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4241562624 bytes 2026-02-18T07:07:50.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4277238784 bytes 2026-02-18T07:07:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4353784832 bytes 2026-02-18T07:07:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4393797632 bytes 2026-02-18T07:07:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4448289792 bytes 2026-02-18T07:07:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4467783680 bytes 2026-02-18T07:07:50.777 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4545196032 bytes 2026-02-18T07:07:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4629151744 bytes 2026-02-18T07:07:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4636551168 bytes 2026-02-18T07:07:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4656196608 bytes 2026-02-18T07:07:50.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4677259264 bytes 2026-02-18T07:07:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4751337472 bytes 2026-02-18T07:07:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4774103040 bytes 2026-02-18T07:07:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4808583168 bytes 2026-02-18T07:07:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4861018112 bytes 2026-02-18T07:07:51.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4949663744 bytes 2026-02-18T07:07:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4986882048 bytes 2026-02-18T07:07:51.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5005874176 bytes 2026-02-18T07:07:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5019018240 bytes 2026-02-18T07:07:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5046167552 bytes 2026-02-18T07:07:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5093971968 bytes 2026-02-18T07:07:51.636 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5125178368 bytes 2026-02-18T07:07:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5181817856 bytes 2026-02-18T07:07:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5198218240 bytes 2026-02-18T07:07:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5340826624 bytes 2026-02-18T07:07:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5382898688 bytes 2026-02-18T07:07:52.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5429445632 bytes 2026-02-18T07:07:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5442813952 bytes 2026-02-18T07:07:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5488396288 bytes 2026-02-18T07:07:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5501896704 bytes 2026-02-18T07:07:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5531704320 bytes 2026-02-18T07:07:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5595523072 bytes 2026-02-18T07:07:52.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5621918720 bytes 2026-02-18T07:07:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5647469568 bytes 2026-02-18T07:07:52.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5710155776 bytes 2026-02-18T07:07:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5710759936 bytes 2026-02-18T07:07:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5781037056 bytes 2026-02-18T07:07:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5846108160 bytes 2026-02-18T07:07:52.910 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5961757696 bytes 2026-02-18T07:07:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6002761728 bytes 2026-02-18T07:07:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6056628224 bytes 2026-02-18T07:07:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6094020608 bytes 2026-02-18T07:07:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6191924224 bytes 2026-02-18T07:07:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6225993728 bytes 2026-02-18T07:07:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6280679424 bytes 2026-02-18T07:07:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6302964736 bytes 2026-02-18T07:07:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6372783104 bytes 2026-02-18T07:07:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6449675264 bytes 2026-02-18T07:07:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6512770048 bytes 2026-02-18T07:07:53.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6528497664 bytes 2026-02-18T07:07:53.677 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6567715840 bytes 2026-02-18T07:07:53.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6654831616 bytes 2026-02-18T07:07:53.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6691241984 bytes 2026-02-18T07:07:53.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6694287360 bytes 2026-02-18T07:07:53.914 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6751221760 bytes 2026-02-18T07:07:53.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6806908928 bytes 2026-02-18T07:07:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6850000896 bytes 2026-02-18T07:07:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6923830272 bytes 2026-02-18T07:07:54.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6939517952 bytes 2026-02-18T07:07:54.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7032404992 bytes 2026-02-18T07:07:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7072125952 bytes 2026-02-18T07:07:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7116505088 bytes 2026-02-18T07:07:54.479 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7190072320 bytes 2026-02-18T07:07:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7264679936 bytes 2026-02-18T07:07:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7275332608 bytes 2026-02-18T07:07:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7347558400 bytes 2026-02-18T07:07:54.755 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7427140608 bytes 2026-02-18T07:07:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7459822592 bytes 2026-02-18T07:07:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7507519488 bytes 2026-02-18T07:07:54.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7565459456 bytes 2026-02-18T07:07:55.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7600076800 bytes 2026-02-18T07:07:55.051 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7625122816 bytes 2026-02-18T07:07:55.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7650059264 bytes 2026-02-18T07:07:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7716745216 bytes 2026-02-18T07:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7756903424 bytes 2026-02-18T07:07:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7803676672 bytes 2026-02-18T07:07:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7804286976 bytes 2026-02-18T07:07:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7828207616 bytes 2026-02-18T07:07:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7828827136 bytes 2026-02-18T07:07:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7842173952 bytes 2026-02-18T07:07:55.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7874890752 bytes 2026-02-18T07:07:55.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7940517888 bytes 2026-02-18T07:07:55.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7975190528 bytes 2026-02-18T07:07:55.695 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8050916352 bytes 2026-02-18T07:07:55.794 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8162089984 bytes 2026-02-18T07:07:55.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8164971520 bytes 2026-02-18T07:07:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8181980160 bytes 2026-02-18T07:07:55.929 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8232888320 bytes 2026-02-18T07:07:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8295615488 bytes 2026-02-18T07:07:56.090 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8366768128 bytes 2026-02-18T07:07:56.129 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-18T07:07:56.129 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-18T07:07:56.129 INFO:tasks.rbd_mirror_thrash:waiting for 5.149447256595913 secs before reviving daemons 2026-02-18T07:07:56.130 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:07:56.129+0000 7f9bad371640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 64976) UID: 1000 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:07:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:07:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=11913205ac85 2026-02-18T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:07:56.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:07:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_7 present 11913205ac85 2026-02-18T07:07:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:07:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:56.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=11913205ac85 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:07:56.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=11913205ac85 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 11913205ac85 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 11913205ac85 -o 11913205ac85 = 11913205ac85 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:07:56.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:07:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-18T07:07:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:07:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:07:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:07:56.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-18T07:07:57.160 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:07:57.159+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fc20000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=192 2026-02-18T07:07:57.161 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:07:57.159+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127fc20000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=193, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=c0, last_copied_object_number=128, snap_seqs={192=18446744073709551614}] 2026-02-18T07:08:01.279 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-18T07:08:01.279 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:08:01.280 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-18T07:08:01.280 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-18T07:08:01.280 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-18T07:08:01.283 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-18T07:08:01.283 INFO:tasks.rbd_mirror_thrash:waiting for 61.65837800462265 secs before thrashing 2026-02-18T07:08:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:08:02.168 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:08:02.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:02.192 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T07:08:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:08:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:08:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:08:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-18T07:08:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_7 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_7 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-18T07:08:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-18T07:08:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:08:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:02.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:02.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:08:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:08:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:05.496 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:08:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:09.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:09.557 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:08:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:17.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:17.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:17.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:17.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:17.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:08:17.820 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.759095520421667) > weight (0.25) 2026-02-18T07:08:17.820 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-18T07:08:17.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-18T07:08:17.821 INFO:tasks.rbd_mirror_thrash:waiting for 6.295297261135859 secs before reviving daemons 2026-02-18T07:08:17.822 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:08:17.821+0000 7f215e4d6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 63863) UID: 1000 2026-02-18T07:08:24.117 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-18T07:08:24.117 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:08:24.118 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-18T07:08:24.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-18T07:08:24.118 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-18T07:08:24.120 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-18T07:08:24.121 INFO:tasks.rbd_mirror_thrash:waiting for 87.59312571875506 secs before thrashing 2026-02-18T07:08:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_7 2026-02-18T07:08:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_7 2026-02-18T07:08:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-18T07:08:33.676 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-18T07:08:33.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_8 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:08:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:08:33.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_8 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:08:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-18T07:08:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:08:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_8': 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1adfb77b3a0b 2026-02-18T07:08:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1adfb77b3a0b 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:08:33 2026 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:08:33 2026 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:08:33 2026 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 1e0cb630-1ba6-4229-83e5-1a4e7b968bc8 2026-02-18T07:08:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:08:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-18T07:08:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:08:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=13 2026-02-18T07:08:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:08:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:08:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 20313088 bytes 2026-02-18T07:08:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 33404928 bytes 2026-02-18T07:08:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 142786560 bytes 2026-02-18T07:08:34.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 213003264 bytes 2026-02-18T07:08:34.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 276375552 bytes 2026-02-18T07:08:34.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 343951360 bytes 2026-02-18T07:08:34.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 399751168 bytes 2026-02-18T07:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 443450368 bytes 2026-02-18T07:08:34.786 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 538167296 bytes 2026-02-18T07:08:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 571203584 bytes 2026-02-18T07:08:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 642914304 bytes 2026-02-18T07:08:34.955 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 671638528 bytes 2026-02-18T07:08:35.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 708071424 bytes 2026-02-18T07:08:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 721205248 bytes 2026-02-18T07:08:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 755975168 bytes 2026-02-18T07:08:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 790987776 bytes 2026-02-18T07:08:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 847860736 bytes 2026-02-18T07:08:35.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 873501696 bytes 2026-02-18T07:08:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 896523264 bytes 2026-02-18T07:08:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 935161856 bytes 2026-02-18T07:08:35.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 944799744 bytes 2026-02-18T07:08:35.486 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 971344896 bytes 2026-02-18T07:08:35.541 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 998438912 bytes 2026-02-18T07:08:35.608 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 1065806848 bytes 2026-02-18T07:08:35.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 1105491968 bytes 2026-02-18T07:08:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1155721216 bytes 2026-02-18T07:08:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1195330560 bytes 2026-02-18T07:08:35.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1238106112 bytes 2026-02-18T07:08:35.899 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1276967936 bytes 2026-02-18T07:08:35.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1315103744 bytes 2026-02-18T07:08:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1364221952 bytes 2026-02-18T07:08:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1438882816 bytes 2026-02-18T07:08:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1449783296 bytes 2026-02-18T07:08:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1493747712 bytes 2026-02-18T07:08:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1542238208 bytes 2026-02-18T07:08:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1606943744 bytes 2026-02-18T07:08:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1645878272 bytes 2026-02-18T07:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1670519808 bytes 2026-02-18T07:08:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1696311296 bytes 2026-02-18T07:08:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1730990080 bytes 2026-02-18T07:08:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1779450880 bytes 2026-02-18T07:08:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1814934528 bytes 2026-02-18T07:08:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1870646272 bytes 2026-02-18T07:08:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1922031616 bytes 2026-02-18T07:08:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 2031949824 bytes 2026-02-18T07:08:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2037390336 bytes 2026-02-18T07:08:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2066073600 bytes 2026-02-18T07:08:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2113880064 bytes 2026-02-18T07:08:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2156058624 bytes 2026-02-18T07:08:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2241442816 bytes 2026-02-18T07:08:37.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2297420800 bytes 2026-02-18T07:08:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2340836352 bytes 2026-02-18T07:08:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2350087168 bytes 2026-02-18T07:08:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2380355584 bytes 2026-02-18T07:08:37.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2414210048 bytes 2026-02-18T07:08:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2448088064 bytes 2026-02-18T07:08:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2479021056 bytes 2026-02-18T07:08:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2517059584 bytes 2026-02-18T07:08:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2550264832 bytes 2026-02-18T07:08:37.806 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2602506240 bytes 2026-02-18T07:08:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2622005248 bytes 2026-02-18T07:08:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2642797568 bytes 2026-02-18T07:08:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2654605312 bytes 2026-02-18T07:08:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2729426944 bytes 2026-02-18T07:08:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2748882944 bytes 2026-02-18T07:08:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2822917120 bytes 2026-02-18T07:08:38.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2851641344 bytes 2026-02-18T07:08:38.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2915032064 bytes 2026-02-18T07:08:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2944978944 bytes 2026-02-18T07:08:38.391 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2945592320 bytes 2026-02-18T07:08:38.429 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2956687360 bytes 2026-02-18T07:08:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 3010466816 bytes 2026-02-18T07:08:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3021117440 bytes 2026-02-18T07:08:38.597 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3071966208 bytes 2026-02-18T07:08:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3136654336 bytes 2026-02-18T07:08:38.710 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3149702144 bytes 2026-02-18T07:08:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3189230592 bytes 2026-02-18T07:08:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3224860672 bytes 2026-02-18T07:08:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3284113408 bytes 2026-02-18T07:08:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3305884672 bytes 2026-02-18T07:08:38.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3306493952 bytes 2026-02-18T07:08:39.025 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3334204416 bytes 2026-02-18T07:08:39.069 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3351829504 bytes 2026-02-18T07:08:39.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3398042624 bytes 2026-02-18T07:08:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3407425536 bytes 2026-02-18T07:08:39.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3438867456 bytes 2026-02-18T07:08:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3502404608 bytes 2026-02-18T07:08:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3534069760 bytes 2026-02-18T07:08:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3570253824 bytes 2026-02-18T07:08:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3654482944 bytes 2026-02-18T07:08:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3688872960 bytes 2026-02-18T07:08:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3742448640 bytes 2026-02-18T07:08:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3774757888 bytes 2026-02-18T07:08:39.765 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3806624768 bytes 2026-02-18T07:08:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3849106432 bytes 2026-02-18T07:08:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3863351296 bytes 2026-02-18T07:08:39.908 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3879545856 bytes 2026-02-18T07:08:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3924319232 bytes 2026-02-18T07:08:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3969775616 bytes 2026-02-18T07:08:40.096 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3998779392 bytes 2026-02-18T07:08:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4025174016 bytes 2026-02-18T07:08:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4086512640 bytes 2026-02-18T07:08:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4168502272 bytes 2026-02-18T07:08:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4251582464 bytes 2026-02-18T07:08:40.455 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4292792320 bytes 2026-02-18T07:08:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4297952256 bytes 2026-02-18T07:08:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4355331072 bytes 2026-02-18T07:08:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4393208832 bytes 2026-02-18T07:08:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4404254720 bytes 2026-02-18T07:08:40.692 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4432861184 bytes 2026-02-18T07:08:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4469529600 bytes 2026-02-18T07:08:40.809 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4516836352 bytes 2026-02-18T07:08:40.881 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4590470144 bytes 2026-02-18T07:08:40.944 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4616213504 bytes 2026-02-18T07:08:40.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4684836864 bytes 2026-02-18T07:08:41.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4795583488 bytes 2026-02-18T07:08:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4848992256 bytes 2026-02-18T07:08:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4872237056 bytes 2026-02-18T07:08:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4927271936 bytes 2026-02-18T07:08:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4950251520 bytes 2026-02-18T07:08:41.422 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4997044224 bytes 2026-02-18T07:08:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 5058799616 bytes 2026-02-18T07:08:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 5122492416 bytes 2026-02-18T07:08:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5181389824 bytes 2026-02-18T07:08:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5300205568 bytes 2026-02-18T07:08:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5325327360 bytes 2026-02-18T07:08:41.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5347343360 bytes 2026-02-18T07:08:41.950 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5435160576 bytes 2026-02-18T07:08:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5469708288 bytes 2026-02-18T07:08:42.048 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5483566080 bytes 2026-02-18T07:08:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5531180032 bytes 2026-02-18T07:08:42.159 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5567032320 bytes 2026-02-18T07:08:42.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5615531008 bytes 2026-02-18T07:08:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5688123392 bytes 2026-02-18T07:08:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5705449472 bytes 2026-02-18T07:08:42.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5758726144 bytes 2026-02-18T07:08:42.482 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5807920128 bytes 2026-02-18T07:08:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5820665856 bytes 2026-02-18T07:08:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5851207680 bytes 2026-02-18T07:08:42.628 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5870811136 bytes 2026-02-18T07:08:42.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5936492544 bytes 2026-02-18T07:08:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5943676928 bytes 2026-02-18T07:08:42.837 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 6047764480 bytes 2026-02-18T07:08:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 6072984576 bytes 2026-02-18T07:08:42.960 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 6102230016 bytes 2026-02-18T07:08:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 6160382976 bytes 2026-02-18T07:08:43.085 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 6192916480 bytes 2026-02-18T07:08:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 6217356288 bytes 2026-02-18T07:08:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 6269237248 bytes 2026-02-18T07:08:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 6327643136 bytes 2026-02-18T07:08:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 6366411776 bytes 2026-02-18T07:08:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 6379703296 bytes 2026-02-18T07:08:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 6429479936 bytes 2026-02-18T07:08:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 6529826816 bytes 2026-02-18T07:08:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 6589972480 bytes 2026-02-18T07:08:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6661292032 bytes 2026-02-18T07:08:43.760 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6661909504 bytes 2026-02-18T07:08:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6684465152 bytes 2026-02-18T07:08:43.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6701024256 bytes 2026-02-18T07:08:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6730682368 bytes 2026-02-18T07:08:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6731314176 bytes 2026-02-18T07:08:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6746488832 bytes 2026-02-18T07:08:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6839501824 bytes 2026-02-18T07:08:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6862405632 bytes 2026-02-18T07:08:44.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6947516416 bytes 2026-02-18T07:08:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6979225600 bytes 2026-02-18T07:08:44.377 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 7033762816 bytes 2026-02-18T07:08:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 7077571584 bytes 2026-02-18T07:08:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 7119210496 bytes 2026-02-18T07:08:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 7149553664 bytes 2026-02-18T07:08:44.672 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 7230697472 bytes 2026-02-18T07:08:44.745 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 7290594304 bytes 2026-02-18T07:08:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 7291211776 bytes 2026-02-18T07:08:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 7392078848 bytes 2026-02-18T07:08:44.938 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 7451110400 bytes 2026-02-18T07:08:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 7492121600 bytes 2026-02-18T07:08:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 7569254400 bytes 2026-02-18T07:08:45.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 7600328704 bytes 2026-02-18T07:08:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7678233600 bytes 2026-02-18T07:08:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7737879552 bytes 2026-02-18T07:08:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7811449856 bytes 2026-02-18T07:08:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7868691456 bytes 2026-02-18T07:08:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7906928640 bytes 2026-02-18T07:08:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7965748224 bytes 2026-02-18T07:08:45.678 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 8021884928 bytes 2026-02-18T07:08:45.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 8083207168 bytes 2026-02-18T07:08:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 8149554176 bytes 2026-02-18T07:08:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 8176872448 bytes 2026-02-18T07:08:45.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 8250781696 bytes 2026-02-18T07:08:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 8271042560 bytes 2026-02-18T07:08:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 8315030528 bytes 2026-02-18T07:08:46.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 8315654144 bytes 2026-02-18T07:08:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 8353953792 bytes 2026-02-18T07:08:46.244 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 8397323264 bytes 2026-02-18T07:08:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 8425293824 bytes 2026-02-18T07:08:46.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 8479876096 bytes 2026-02-18T07:08:46.421 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 8499705856 bytes 2026-02-18T07:08:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 8539556864 bytes 2026-02-18T07:08:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 8581264384 bytes 2026-02-18T07:08:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8588771328 bytes 2026-02-18T07:08:46.649 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20100 IOs, 8637220864 bytes 2026-02-18T07:08:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20200 IOs, 8637846528 bytes 2026-02-18T07:08:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20300 IOs, 8668539904 bytes 2026-02-18T07:08:46.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20400 IOs, 8722036736 bytes 2026-02-18T07:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20500 IOs, 8766420992 bytes 2026-02-18T07:08:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20600 IOs, 8768178176 bytes 2026-02-18T07:08:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:08:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:08:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:08:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:08:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=11912eb85675 2026-02-18T07:08:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:08:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:08:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_8 present 11912eb85675 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=11912eb85675 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:08:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:08:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=11912eb85675 2026-02-18T07:08:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 11912eb85675 2026-02-18T07:08:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 11912eb85675 -o 11912eb85675 = 11912eb85675 2026-02-18T07:08:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:08:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-18T07:08:48.154 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:08:48.152+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127ffb8000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2026-02-18T07:08:48.155 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:08:48.152+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127ffb8000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2026-02-18T07:08:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:08:53.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-18T07:08:53.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:08:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:08:53.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T07:08:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:08:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:08:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_8 2026-02-18T07:08:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:08:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:08:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_8 2026-02-18T07:08:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-18T07:08:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:08:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-18T07:08:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-18T07:08:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-18T07:08:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:08:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:08:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:08:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:08:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:08:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:08:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:08:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:08:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:08:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:08:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:08:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:08:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:08:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:08:56.491 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:08:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:08:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:08:56.518 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:08:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:08:56.519 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:09:00.520 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:09:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:09:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:09:00.552 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:09:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:09:02.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.30415513931609295) > weight (0.25) 2026-02-18T07:09:02.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.49115432232726075) > weight (0.25) 2026-02-18T07:09:02.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.31901503417005916) > weight (0.25) 2026-02-18T07:09:02.943 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9279229044093175) > weight (0.25) 2026-02-18T07:09:02.943 INFO:tasks.rbd_mirror_thrash:waiting for 68.00028719788463 secs before thrashing 2026-02-18T07:09:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:09:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:09:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:09:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:09:08.844 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:08.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:08.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:09:24.847 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_8 2026-02-18T07:09:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_8 2026-02-18T07:09:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-18T07:09:24.880 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-18T07:09:24.993 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:09:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_9 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:09:24.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:09:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:09:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:09:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_9 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:09:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-18T07:09:25.102 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:09:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-18T07:09:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_9': 2026-02-18T07:09:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:09:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1b15bb8ff726 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1b15bb8ff726 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:09:25 2026 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:09:25 2026 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:09:25 2026 2026-02-18T07:09:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:09:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:09:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 706e52e3-43cf-46f3-9b68-ce56cf29ab7f 2026-02-18T07:09:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:09:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-18T07:09:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:09:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=9 2026-02-18T07:09:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:09:25.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:09:25.222 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 34681856 bytes 2026-02-18T07:09:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 118311936 bytes 2026-02-18T07:09:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 179191808 bytes 2026-02-18T07:09:25.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 205691904 bytes 2026-02-18T07:09:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 268446720 bytes 2026-02-18T07:09:25.571 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 320737280 bytes 2026-02-18T07:09:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 345079808 bytes 2026-02-18T07:09:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 377313280 bytes 2026-02-18T07:09:25.704 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 395043840 bytes 2026-02-18T07:09:25.751 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 411752448 bytes 2026-02-18T07:09:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 466882560 bytes 2026-02-18T07:09:25.857 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 482177024 bytes 2026-02-18T07:09:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 490351616 bytes 2026-02-18T07:09:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 521189376 bytes 2026-02-18T07:09:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 538338304 bytes 2026-02-18T07:09:26.020 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 556338176 bytes 2026-02-18T07:09:26.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 612709376 bytes 2026-02-18T07:09:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 624050176 bytes 2026-02-18T07:09:26.189 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 662902784 bytes 2026-02-18T07:09:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 757364736 bytes 2026-02-18T07:09:26.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 770336768 bytes 2026-02-18T07:09:26.418 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 855675904 bytes 2026-02-18T07:09:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 916192256 bytes 2026-02-18T07:09:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 963609600 bytes 2026-02-18T07:09:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 999376896 bytes 2026-02-18T07:09:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 1009190912 bytes 2026-02-18T07:09:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 1023863808 bytes 2026-02-18T07:09:26.761 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 1094994944 bytes 2026-02-18T07:09:26.819 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 1145356288 bytes 2026-02-18T07:09:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 1171466240 bytes 2026-02-18T07:09:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 1176064000 bytes 2026-02-18T07:09:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1247075328 bytes 2026-02-18T07:09:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1269316608 bytes 2026-02-18T07:09:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1318236160 bytes 2026-02-18T07:09:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1353146368 bytes 2026-02-18T07:09:27.221 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1400458240 bytes 2026-02-18T07:09:27.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1480814592 bytes 2026-02-18T07:09:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1537498112 bytes 2026-02-18T07:09:27.434 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1551237120 bytes 2026-02-18T07:09:27.499 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1614920704 bytes 2026-02-18T07:09:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1671756800 bytes 2026-02-18T07:09:27.645 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1725032448 bytes 2026-02-18T07:09:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1765715968 bytes 2026-02-18T07:09:27.746 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1793910784 bytes 2026-02-18T07:09:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1892645888 bytes 2026-02-18T07:09:27.943 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 2014455808 bytes 2026-02-18T07:09:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 2015069184 bytes 2026-02-18T07:09:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 2031187968 bytes 2026-02-18T07:09:28.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 2066327552 bytes 2026-02-18T07:09:28.146 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 2114599936 bytes 2026-02-18T07:09:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 2173616128 bytes 2026-02-18T07:09:28.249 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 2186745856 bytes 2026-02-18T07:09:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 2204379136 bytes 2026-02-18T07:09:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 2226667520 bytes 2026-02-18T07:09:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2243272704 bytes 2026-02-18T07:09:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2302694400 bytes 2026-02-18T07:09:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2324968448 bytes 2026-02-18T07:09:28.772 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2392067072 bytes 2026-02-18T07:09:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2416089088 bytes 2026-02-18T07:09:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2460682240 bytes 2026-02-18T07:09:28.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2586585088 bytes 2026-02-18T07:09:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2598394880 bytes 2026-02-18T07:09:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2642108416 bytes 2026-02-18T07:09:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2670802944 bytes 2026-02-18T07:09:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2693271552 bytes 2026-02-18T07:09:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2722118656 bytes 2026-02-18T07:09:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2761528320 bytes 2026-02-18T07:09:29.364 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2824683520 bytes 2026-02-18T07:09:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2866182144 bytes 2026-02-18T07:09:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2902633472 bytes 2026-02-18T07:09:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2970335232 bytes 2026-02-18T07:09:29.602 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2988101632 bytes 2026-02-18T07:09:29.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 3021944832 bytes 2026-02-18T07:09:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 3025555456 bytes 2026-02-18T07:09:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 3026157568 bytes 2026-02-18T07:09:29.769 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 3069598720 bytes 2026-02-18T07:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 3076530176 bytes 2026-02-18T07:09:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 3088276480 bytes 2026-02-18T07:09:29.884 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 3109373952 bytes 2026-02-18T07:09:29.932 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 3128599552 bytes 2026-02-18T07:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3165208576 bytes 2026-02-18T07:09:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3266512896 bytes 2026-02-18T07:09:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3282637824 bytes 2026-02-18T07:09:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3353758720 bytes 2026-02-18T07:09:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3393189888 bytes 2026-02-18T07:09:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3404086272 bytes 2026-02-18T07:09:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3453356032 bytes 2026-02-18T07:09:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3492408320 bytes 2026-02-18T07:09:30.506 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3590970368 bytes 2026-02-18T07:09:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3608052736 bytes 2026-02-18T07:09:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3618642944 bytes 2026-02-18T07:09:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3622833152 bytes 2026-02-18T07:09:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3698355200 bytes 2026-02-18T07:09:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3780597760 bytes 2026-02-18T07:09:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3842322432 bytes 2026-02-18T07:09:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3857038336 bytes 2026-02-18T07:09:30.951 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3898645504 bytes 2026-02-18T07:09:31.016 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3939597312 bytes 2026-02-18T07:09:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 4003606528 bytes 2026-02-18T07:09:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 4014792704 bytes 2026-02-18T07:09:31.174 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 4054557696 bytes 2026-02-18T07:09:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 4130563072 bytes 2026-02-18T07:09:31.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 4160995328 bytes 2026-02-18T07:09:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 4161616896 bytes 2026-02-18T07:09:31.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 4214577152 bytes 2026-02-18T07:09:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 4265175040 bytes 2026-02-18T07:09:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 4310647808 bytes 2026-02-18T07:09:31.606 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4352300032 bytes 2026-02-18T07:09:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4439068672 bytes 2026-02-18T07:09:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4480257024 bytes 2026-02-18T07:09:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4487676928 bytes 2026-02-18T07:09:31.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4520383488 bytes 2026-02-18T07:09:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4593338368 bytes 2026-02-18T07:09:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4602571776 bytes 2026-02-18T07:09:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4645685248 bytes 2026-02-18T07:09:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4663305216 bytes 2026-02-18T07:09:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4728776704 bytes 2026-02-18T07:09:32.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4749516800 bytes 2026-02-18T07:09:32.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4766473216 bytes 2026-02-18T07:09:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4773150720 bytes 2026-02-18T07:09:32.545 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4838312960 bytes 2026-02-18T07:09:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4950484992 bytes 2026-02-18T07:09:32.699 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4989538304 bytes 2026-02-18T07:09:32.743 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 5009484800 bytes 2026-02-18T07:09:32.811 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 5065051136 bytes 2026-02-18T07:09:32.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 5089155072 bytes 2026-02-18T07:09:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 5136602112 bytes 2026-02-18T07:09:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 5169779712 bytes 2026-02-18T07:09:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 5208158208 bytes 2026-02-18T07:09:33.066 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 5221266432 bytes 2026-02-18T07:09:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 5256168448 bytes 2026-02-18T07:09:33.175 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5297389568 bytes 2026-02-18T07:09:33.205 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5301316608 bytes 2026-02-18T07:09:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5329696768 bytes 2026-02-18T07:09:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5330283520 bytes 2026-02-18T07:09:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5398196224 bytes 2026-02-18T07:09:33.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5452777472 bytes 2026-02-18T07:09:33.501 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5513702400 bytes 2026-02-18T07:09:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5577662464 bytes 2026-02-18T07:09:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5603980288 bytes 2026-02-18T07:09:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5609850880 bytes 2026-02-18T07:09:33.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5698970624 bytes 2026-02-18T07:09:33.782 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5700003840 bytes 2026-02-18T07:09:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5757176832 bytes 2026-02-18T07:09:33.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5781757952 bytes 2026-02-18T07:09:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5833942016 bytes 2026-02-18T07:09:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5834557440 bytes 2026-02-18T07:09:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5863214080 bytes 2026-02-18T07:09:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5898602496 bytes 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:09:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:09:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=1191edd24166 2026-02-18T07:09:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:09:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_9 present 1191edd24166 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:09:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=1191edd24166 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:09:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:09:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=1191edd24166 2026-02-18T07:09:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1191edd24166 2026-02-18T07:09:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 1191edd24166 -o 1191edd24166 = 1191edd24166 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-18T07:09:35.102 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:09:35.100+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f754000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=194 2026-02-18T07:09:35.102 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:09:35.100+0000 7f835350c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f754000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=195, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=c2, last_copied_object_number=128, snap_seqs={194=18446744073709551614}] 2026-02-18T07:09:38.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:09:38.137+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d27462000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:09:38.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:09:38.137+0000 7ff795cd6640 -1 rbd::mirror::ImageReplayer: 0x560d27803900 [3/706e52e3-43cf-46f3-9b68-ce56cf29ab7f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:09:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:09:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-18T07:09:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:09:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:09:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:09:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T07:09:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:09:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:09:40.137 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:09:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-18T07:09:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_9 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_9 2026-02-18T07:09:40.240 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-18T07:09:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:09:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-18T07:09:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:09:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:09:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:40.264 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-18T07:09:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-18T07:09:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:40.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:40.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:09:40.293 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:09:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:09:40.322 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:09:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:09:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:09:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:09:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:09:41.404 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:41.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:41.405 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:09:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:09:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:09:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:09:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:43.463 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:09:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:09:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:47.465 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:09:47.495 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:09:47.521 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:47.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:47.523 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:09:51.714 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7288542494241421) > weight (0.25) 2026-02-18T07:09:51.715 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6592932865806435) > weight (0.25) 2026-02-18T07:09:51.715 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7028762437193767) > weight (0.25) 2026-02-18T07:09:51.715 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6070983315609861) > weight (0.25) 2026-02-18T07:09:51.715 INFO:tasks.rbd_mirror_thrash:waiting for 72.13784335120724 secs before thrashing 2026-02-18T07:09:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:09:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:09:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:09:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:09:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:09:55.555 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:09:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:09:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:09:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:09:55.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:09:55.584 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:10:10.944 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6904060290094138) > weight (0.25) 2026-02-18T07:10:10.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9761726358132621) > weight (0.25) 2026-02-18T07:10:10.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9380916129712034) > weight (0.25) 2026-02-18T07:10:10.945 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7988827690484784) > weight (0.25) 2026-02-18T07:10:10.945 INFO:tasks.rbd_mirror_thrash:waiting for 102.24217912982598 secs before thrashing 2026-02-18T07:10:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_9 2026-02-18T07:10:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_9 2026-02-18T07:10:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-18T07:10:11.616 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-18T07:10:11.920 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:10:11.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:10:11.922 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for i in `seq 1 10` 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image=image_10 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local size=128 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -n 512M ']' 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ size=512M 2026-02-18T07:10:11.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ shift 2026-02-18T07:10:11.925 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering mirror/image_10 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local mode=snapshot 2026-02-18T07:10:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-18T07:10:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring enabled 2026-02-18T07:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-18T07:10:12.132 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:rbd image 'image_10': 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: size 512 MiB in 128 objects 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: order 22 (4 MiB objects) 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: snapshot_count: 1 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: id: 1b4b30605816 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: block_name_prefix: rbd_data.1b4b30605816 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: format: 2 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: features: layering 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: op_features: 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: flags: 2026-02-18T07:10:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: create_timestamp: Wed Feb 18 07:10:11 2026 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: access_timestamp: Wed Feb 18 07:10:11 2026 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: modify_timestamp: Wed Feb 18 07:10:11 2026 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring state: enabled 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring mode: snapshot 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring global id: 7583c316-1d73-48e2-ae75-d26c337492c2 2026-02-18T07:10:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout: mirroring primary: true 2026-02-18T07:10:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-18T07:10:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:12.135 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-18T07:10:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local duration=12 2026-02-18T07:10:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:10:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-18T07:10:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:100 IOs, 24940544 bytes 2026-02-18T07:10:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:200 IOs, 67452928 bytes 2026-02-18T07:10:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:300 IOs, 82213888 bytes 2026-02-18T07:10:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:400 IOs, 82852864 bytes 2026-02-18T07:10:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:500 IOs, 92213248 bytes 2026-02-18T07:10:12.424 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:600 IOs, 109446144 bytes 2026-02-18T07:10:12.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:700 IOs, 123541504 bytes 2026-02-18T07:10:12.507 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:800 IOs, 141710336 bytes 2026-02-18T07:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:900 IOs, 225726464 bytes 2026-02-18T07:10:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1000 IOs, 238445568 bytes 2026-02-18T07:10:12.670 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1100 IOs, 249563136 bytes 2026-02-18T07:10:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1200 IOs, 362833920 bytes 2026-02-18T07:10:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1300 IOs, 429664256 bytes 2026-02-18T07:10:12.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1400 IOs, 439707648 bytes 2026-02-18T07:10:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1500 IOs, 463447040 bytes 2026-02-18T07:10:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1600 IOs, 524748800 bytes 2026-02-18T07:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1700 IOs, 561165312 bytes 2026-02-18T07:10:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1800 IOs, 587631616 bytes 2026-02-18T07:10:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:1900 IOs, 618752000 bytes 2026-02-18T07:10:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2000 IOs, 640459776 bytes 2026-02-18T07:10:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2100 IOs, 676781056 bytes 2026-02-18T07:10:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2200 IOs, 701279232 bytes 2026-02-18T07:10:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2300 IOs, 726929408 bytes 2026-02-18T07:10:13.413 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2400 IOs, 738381824 bytes 2026-02-18T07:10:13.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2500 IOs, 773658624 bytes 2026-02-18T07:10:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2600 IOs, 794627072 bytes 2026-02-18T07:10:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2700 IOs, 827161600 bytes 2026-02-18T07:10:13.642 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2800 IOs, 877914112 bytes 2026-02-18T07:10:13.690 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:2900 IOs, 898898944 bytes 2026-02-18T07:10:13.733 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3000 IOs, 928431104 bytes 2026-02-18T07:10:13.789 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3100 IOs, 963706880 bytes 2026-02-18T07:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3200 IOs, 1003017216 bytes 2026-02-18T07:10:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3300 IOs, 1079428096 bytes 2026-02-18T07:10:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3400 IOs, 1116125184 bytes 2026-02-18T07:10:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3500 IOs, 1155554304 bytes 2026-02-18T07:10:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3600 IOs, 1242967040 bytes 2026-02-18T07:10:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3700 IOs, 1246394368 bytes 2026-02-18T07:10:14.199 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3800 IOs, 1248153600 bytes 2026-02-18T07:10:14.250 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:3900 IOs, 1278578688 bytes 2026-02-18T07:10:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4000 IOs, 1369354240 bytes 2026-02-18T07:10:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4100 IOs, 1399185408 bytes 2026-02-18T07:10:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4200 IOs, 1428902912 bytes 2026-02-18T07:10:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4300 IOs, 1537337344 bytes 2026-02-18T07:10:14.627 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4400 IOs, 1590789120 bytes 2026-02-18T07:10:14.663 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4500 IOs, 1600774144 bytes 2026-02-18T07:10:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4600 IOs, 1670715392 bytes 2026-02-18T07:10:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4700 IOs, 1697985536 bytes 2026-02-18T07:10:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4800 IOs, 1721684992 bytes 2026-02-18T07:10:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:4900 IOs, 1801662464 bytes 2026-02-18T07:10:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5000 IOs, 1807337472 bytes 2026-02-18T07:10:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5100 IOs, 1842570240 bytes 2026-02-18T07:10:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5200 IOs, 1880254464 bytes 2026-02-18T07:10:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5300 IOs, 1932362752 bytes 2026-02-18T07:10:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5400 IOs, 1971546112 bytes 2026-02-18T07:10:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5500 IOs, 2057479168 bytes 2026-02-18T07:10:15.324 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5600 IOs, 2094455808 bytes 2026-02-18T07:10:15.399 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5700 IOs, 2140158976 bytes 2026-02-18T07:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5800 IOs, 2167580672 bytes 2026-02-18T07:10:15.509 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:5900 IOs, 2202061824 bytes 2026-02-18T07:10:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6000 IOs, 2243382272 bytes 2026-02-18T07:10:15.612 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6100 IOs, 2264235008 bytes 2026-02-18T07:10:15.673 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6200 IOs, 2309369856 bytes 2026-02-18T07:10:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6300 IOs, 2342941696 bytes 2026-02-18T07:10:15.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6400 IOs, 2404905984 bytes 2026-02-18T07:10:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6500 IOs, 2439261184 bytes 2026-02-18T07:10:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6600 IOs, 2449308672 bytes 2026-02-18T07:10:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6700 IOs, 2515557376 bytes 2026-02-18T07:10:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6800 IOs, 2519729152 bytes 2026-02-18T07:10:16.065 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:6900 IOs, 2567767040 bytes 2026-02-18T07:10:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7000 IOs, 2585689088 bytes 2026-02-18T07:10:16.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7100 IOs, 2631858176 bytes 2026-02-18T07:10:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7200 IOs, 2675640320 bytes 2026-02-18T07:10:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7300 IOs, 2747025408 bytes 2026-02-18T07:10:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7400 IOs, 2756666368 bytes 2026-02-18T07:10:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7500 IOs, 2820990976 bytes 2026-02-18T07:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7600 IOs, 2885363712 bytes 2026-02-18T07:10:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7700 IOs, 2897088512 bytes 2026-02-18T07:10:16.622 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7800 IOs, 2935844864 bytes 2026-02-18T07:10:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:7900 IOs, 2954775552 bytes 2026-02-18T07:10:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8000 IOs, 2999652352 bytes 2026-02-18T07:10:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8100 IOs, 3097285632 bytes 2026-02-18T07:10:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8200 IOs, 3116472320 bytes 2026-02-18T07:10:16.947 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8300 IOs, 3158697984 bytes 2026-02-18T07:10:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8400 IOs, 3159300096 bytes 2026-02-18T07:10:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8500 IOs, 3182441472 bytes 2026-02-18T07:10:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8600 IOs, 3282163712 bytes 2026-02-18T07:10:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8700 IOs, 3330539520 bytes 2026-02-18T07:10:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8800 IOs, 3331168256 bytes 2026-02-18T07:10:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:8900 IOs, 3383174144 bytes 2026-02-18T07:10:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9000 IOs, 3423643648 bytes 2026-02-18T07:10:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9100 IOs, 3430355968 bytes 2026-02-18T07:10:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9200 IOs, 3498578944 bytes 2026-02-18T07:10:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9300 IOs, 3549305856 bytes 2026-02-18T07:10:17.567 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9400 IOs, 3549916160 bytes 2026-02-18T07:10:17.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9500 IOs, 3614175232 bytes 2026-02-18T07:10:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9600 IOs, 3690627072 bytes 2026-02-18T07:10:17.779 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9700 IOs, 3720928256 bytes 2026-02-18T07:10:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9800 IOs, 3729161216 bytes 2026-02-18T07:10:17.879 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:9900 IOs, 3772317696 bytes 2026-02-18T07:10:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10000 IOs, 3806243840 bytes 2026-02-18T07:10:17.961 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10100 IOs, 3806869504 bytes 2026-02-18T07:10:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10200 IOs, 3809630208 bytes 2026-02-18T07:10:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10300 IOs, 3882850304 bytes 2026-02-18T07:10:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10400 IOs, 3884855296 bytes 2026-02-18T07:10:18.161 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10500 IOs, 3933898752 bytes 2026-02-18T07:10:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10600 IOs, 3983073280 bytes 2026-02-18T07:10:18.267 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10700 IOs, 3984052224 bytes 2026-02-18T07:10:18.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10800 IOs, 4057970688 bytes 2026-02-18T07:10:18.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:10900 IOs, 4117690368 bytes 2026-02-18T07:10:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11000 IOs, 4155159552 bytes 2026-02-18T07:10:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11100 IOs, 4192272384 bytes 2026-02-18T07:10:18.603 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11200 IOs, 4230734848 bytes 2026-02-18T07:10:18.632 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11300 IOs, 4231331840 bytes 2026-02-18T07:10:18.683 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11400 IOs, 4267502592 bytes 2026-02-18T07:10:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11500 IOs, 4314924032 bytes 2026-02-18T07:10:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11600 IOs, 4336652288 bytes 2026-02-18T07:10:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11700 IOs, 4404320256 bytes 2026-02-18T07:10:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11800 IOs, 4494982144 bytes 2026-02-18T07:10:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:11900 IOs, 4542386176 bytes 2026-02-18T07:10:19.118 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12000 IOs, 4597249024 bytes 2026-02-18T07:10:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12100 IOs, 4618758144 bytes 2026-02-18T07:10:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12200 IOs, 4675460096 bytes 2026-02-18T07:10:19.310 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12300 IOs, 4737570816 bytes 2026-02-18T07:10:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12400 IOs, 4752193536 bytes 2026-02-18T07:10:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12500 IOs, 4765587456 bytes 2026-02-18T07:10:19.456 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12600 IOs, 4815212544 bytes 2026-02-18T07:10:19.489 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12700 IOs, 4829594624 bytes 2026-02-18T07:10:19.543 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12800 IOs, 4848528384 bytes 2026-02-18T07:10:19.598 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:12900 IOs, 4874084352 bytes 2026-02-18T07:10:19.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13000 IOs, 4901292032 bytes 2026-02-18T07:10:19.728 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13100 IOs, 4961075200 bytes 2026-02-18T07:10:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13200 IOs, 5004491776 bytes 2026-02-18T07:10:19.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13300 IOs, 5039322112 bytes 2026-02-18T07:10:19.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13400 IOs, 5109011456 bytes 2026-02-18T07:10:19.989 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13500 IOs, 5182507008 bytes 2026-02-18T07:10:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13600 IOs, 5183121408 bytes 2026-02-18T07:10:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13700 IOs, 5232109568 bytes 2026-02-18T07:10:20.165 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13800 IOs, 5287762944 bytes 2026-02-18T07:10:20.195 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:13900 IOs, 5294471168 bytes 2026-02-18T07:10:20.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14000 IOs, 5335087104 bytes 2026-02-18T07:10:20.298 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14100 IOs, 5370420224 bytes 2026-02-18T07:10:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14200 IOs, 5408379904 bytes 2026-02-18T07:10:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14300 IOs, 5426503680 bytes 2026-02-18T07:10:20.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14400 IOs, 5478340608 bytes 2026-02-18T07:10:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14500 IOs, 5512300544 bytes 2026-02-18T07:10:20.634 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14600 IOs, 5590986752 bytes 2026-02-18T07:10:20.689 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14700 IOs, 5632674816 bytes 2026-02-18T07:10:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14800 IOs, 5669957632 bytes 2026-02-18T07:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:14900 IOs, 5687889920 bytes 2026-02-18T07:10:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15000 IOs, 5700686848 bytes 2026-02-18T07:10:20.907 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15100 IOs, 5745205248 bytes 2026-02-18T07:10:20.982 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15200 IOs, 5824416768 bytes 2026-02-18T07:10:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15300 IOs, 5871622144 bytes 2026-02-18T07:10:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15400 IOs, 5946718208 bytes 2026-02-18T07:10:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15500 IOs, 5971966976 bytes 2026-02-18T07:10:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15600 IOs, 6031418368 bytes 2026-02-18T07:10:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15700 IOs, 6065030144 bytes 2026-02-18T07:10:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15800 IOs, 6078187520 bytes 2026-02-18T07:10:21.440 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:15900 IOs, 6114583552 bytes 2026-02-18T07:10:21.494 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16000 IOs, 6159512576 bytes 2026-02-18T07:10:21.532 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16100 IOs, 6160107520 bytes 2026-02-18T07:10:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16200 IOs, 6250912768 bytes 2026-02-18T07:10:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16300 IOs, 6315592704 bytes 2026-02-18T07:10:21.766 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16400 IOs, 6355500032 bytes 2026-02-18T07:10:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16500 IOs, 6411106304 bytes 2026-02-18T07:10:21.888 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16600 IOs, 6430782464 bytes 2026-02-18T07:10:21.926 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16700 IOs, 6472604672 bytes 2026-02-18T07:10:22.037 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16800 IOs, 6570271744 bytes 2026-02-18T07:10:22.101 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:16900 IOs, 6610587648 bytes 2026-02-18T07:10:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17000 IOs, 6650283008 bytes 2026-02-18T07:10:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17100 IOs, 6760730624 bytes 2026-02-18T07:10:22.304 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17200 IOs, 6770453504 bytes 2026-02-18T07:10:22.337 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17300 IOs, 6771065856 bytes 2026-02-18T07:10:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17400 IOs, 6804768768 bytes 2026-02-18T07:10:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17500 IOs, 6886140928 bytes 2026-02-18T07:10:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17600 IOs, 6944203776 bytes 2026-02-18T07:10:22.589 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17700 IOs, 6954692608 bytes 2026-02-18T07:10:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17800 IOs, 6997007360 bytes 2026-02-18T07:10:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:17900 IOs, 7027552256 bytes 2026-02-18T07:10:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18000 IOs, 7039007744 bytes 2026-02-18T07:10:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18100 IOs, 7107164160 bytes 2026-02-18T07:10:22.875 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18200 IOs, 7145945088 bytes 2026-02-18T07:10:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18300 IOs, 7151763456 bytes 2026-02-18T07:10:23.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18400 IOs, 7231185920 bytes 2026-02-18T07:10:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18500 IOs, 7315312640 bytes 2026-02-18T07:10:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18600 IOs, 7391420416 bytes 2026-02-18T07:10:23.234 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18700 IOs, 7425234944 bytes 2026-02-18T07:10:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18800 IOs, 7474743296 bytes 2026-02-18T07:10:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:18900 IOs, 7522777088 bytes 2026-02-18T07:10:23.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19000 IOs, 7539918848 bytes 2026-02-18T07:10:23.441 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19100 IOs, 7540531200 bytes 2026-02-18T07:10:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19200 IOs, 7545563136 bytes 2026-02-18T07:10:23.570 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19300 IOs, 7619562496 bytes 2026-02-18T07:10:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19400 IOs, 7636920320 bytes 2026-02-18T07:10:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19500 IOs, 7665546240 bytes 2026-02-18T07:10:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19600 IOs, 7730141184 bytes 2026-02-18T07:10:23.831 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19700 IOs, 7815391232 bytes 2026-02-18T07:10:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19800 IOs, 7903430656 bytes 2026-02-18T07:10:24.030 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:19900 IOs, 7988321280 bytes 2026-02-18T07:10:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:20000 IOs, 8049464320 bytes 2026-02-18T07:10:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=124 2026-02-18T07:10:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:10:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 124 -eq 124 ']' 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=present 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:10:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:10:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:10:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:10:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:10:24.143 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:10:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id=119197d73f14 2026-02-18T07:10:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:10:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:10:24.271 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_10 present 119197d73f14 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=present 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id=119197d73f14 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:10:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:10:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:10:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:10:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id=119197d73f14 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 119197d73f14 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -z 119197d73f14 -o 119197d73f14 = 119197d73f14 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_state=present 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test present = present 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:10:24.301 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-18T07:10:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-18T07:10:25.160 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:10:25.158+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f5f9800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=195 2026-02-18T07:10:25.160 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:10:25.158+0000 7f8353d0d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x56127f5f9800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=196, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=19836290-025c-408f-8a50-82436ee28aa0, primary_snap_id=c3, last_copied_object_number=128, snap_seqs={195=18446744073709551614}] 2026-02-18T07:10:26.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:10:26.021+0000 7ff78fcca640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x560d271ffd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-18T07:10:26.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:10:26.021+0000 7ff7964d7640 -1 rbd::mirror::ImageReplayer: 0x560d27803900 [3/7583c316-1d73-48e2-ae75-d26c337492c2] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-18T07:10:30.164 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:Mirroring disabled 2026-02-18T07:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-18T07:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local state=deleted 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local s 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:10:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T07:10:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ image_id= 2026-02-18T07:10:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-18T07:10:30.198 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0.1 2026-02-18T07:10:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-18T07:10:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:10:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local test_state=deleted 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image_id= 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_state=deleted 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local current_image_id 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ get_image_id cluster1 mirror image_10 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local image=image_10 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-18T07:10:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-18T07:10:30.323 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-18T07:10:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ current_image_id= 2026-02-18T07:10:30.325 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n '' 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test deleted = deleted 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:10:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:10:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:10:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 1 2026-02-18T07:10:31.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:10:31.411 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:31.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:31.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:31.412 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:10:31.439 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:10:31.466 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:10:31.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:10:31.467 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:10:31.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:31.468 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 2 2026-02-18T07:10:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:10:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:10:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:10:33.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:10:33.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:10:33.526 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:10:33.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:33.527 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 4 2026-02-18T07:10:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:10:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:10:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:10:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:10:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:10:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:10:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 8 2026-02-18T07:10:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:10:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:10:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:10:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:10:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:10:45.621 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:10:45.852 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 0% complete...failed. 2026-02-18T07:10:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:rbd: error: image still has watchers 2026-02-18T07:10:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed client to timeout. 2026-02-18T07:10:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-18T07:10:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 16 2026-02-18T07:11:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ remove_image cluster2 mirror image_10 2026-02-18T07:11:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local image=image_10 2026-02-18T07:11:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-18T07:11:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-18T07:11:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-18T07:11:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-18T07:11:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ tee -a /tmp/tmp.y3X31uzVRt/rbd-mirror.test.log 2026-02-18T07:11:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ date '+%F %T' 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo 2026-02-18 07:11:02 TEST: check if removed 'images'\''' OMAP are removed 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:2026-02-18 07:11:02 TEST: check if removed images' OMAP are removed 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirroring 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=status_global 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.004 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=status_global 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-18T07:11:02.005 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c status_global 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirroring 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=image_ 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=image_ 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.027 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c image_ 2026-02-18T07:11:02.028 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=image_map 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=image_map 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster1 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c image_map 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.253 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirroring 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=status_global 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.274 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=status_global 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c status_global 2026-02-18T07:11:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirroring 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=image_ 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.296 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=image_ 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirroring 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-18T07:11:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c image_ 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local pool=mirror 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local filter=image_map 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ sleep 0 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local filter=image_map 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local cluster=cluster2 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local pool=mirror 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ local obj_name=rbd_mirror_leader 2026-02-18T07:11:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-18T07:11:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ grep -c image_map 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test 0 = 0 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ error_code=0 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set -e 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 0 2026-02-18T07:11:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ cleanup 0 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local error_code=0 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ set +e 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -ne 0 ']' 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z '' ']' 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster instance 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T07:11:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-18T07:11:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror' does not exist 2026-02-18T07:11:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T07:11:03.087 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-18T07:11:03.853 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6601014525887686) > weight (0.25) 2026-02-18T07:11:03.854 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8825549313268117) > weight (0.25) 2026-02-18T07:11:03.854 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-18T07:11:03.854 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-18T07:11:03.854 INFO:tasks.rbd_mirror_thrash:waiting for 4.068989622992367 secs before reviving daemons 2026-02-18T07:11:03.855 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.854+0000 7f89586f2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 62838) UID: 1000 2026-02-18T07:11:03.856 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f8957ef1640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2b64000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5886 object: (2) No such file or directory 2026-02-18T07:11:03.856 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f8957ef1640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2b65c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5886 object: (2) No such file or directory 2026-02-18T07:11:03.856 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f89576f0640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2883180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5886 object: (2) No such file or directory 2026-02-18T07:11:03.856 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f8950ee3640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2b64000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:03.857 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f8950ee3640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2b65c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:03.857 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.855+0000 7f89516e4640 -1 rbd::mirror::InstanceWatcher: 0x55bfc2883180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f8950ee3640 -1 rbd::mirror::MirrorStatusUpdater 0x55bfc1e4a360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f8957ef1640 -1 rbd::mirror::NamespaceReplayer: 0x55bfc0dc6700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f89516e4640 -1 rbd::mirror::MirrorStatusUpdater 0x55bfc1e4a480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f89576f0640 -1 rbd::mirror::NamespaceReplayer: 0x55bfc0dc6a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f8950ee3640 -1 rbd::mirror::MirrorStatusUpdater 0x55bfc1e4a240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:03.861 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.859+0000 7f8957ef1640 -1 rbd::mirror::NamespaceReplayer: 0x55bfc0dc6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:03.864 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:03.863+0000 7f89576f0640 -1 rbd::mirror::LeaderWatcher: 0x55bfc27f6300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror' does not exist 2026-02-18T07:11:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T07:11:04.208 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-18T07:11:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror_parent' does not exist 2026-02-18T07:11:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ CEPH_ARGS= 2026-02-18T07:11:05.091 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-18T07:11:05.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.297+0000 7f8360526640 -1 rbd::mirror::LeaderWatcher: 0x56127ea83b00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-18T07:11:05.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.297+0000 7f834ccff640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x56127f68c6c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:05.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.297+0000 7f834ccff640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x56127f68cd80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:05.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.297+0000 7f834c4fe640 -1 rbd::mirror::PoolWatcher: 0x56127ef1d4a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.301 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.297+0000 7f834ccff640 -1 rbd::mirror::PoolWatcher: 0x56127ef1de00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.299+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127ea83b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T07:11:05.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.299+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127ea83b00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.302+0000 7f8360526640 -1 rbd::mirror::LeaderWatcher: 0x56127f4da300 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.302+0000 7f8359d19640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x56127f07efc0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f835a51a640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x56127f68c000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f835a51a640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x56127f07f440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8359d19640 -1 rbd::mirror::PoolWatcher: 0x56127ef303c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.306 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8353d0d640 -1 rbd::mirror::PoolWatcher: 0x56127f7ec000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8353d0d640 -1 rbd::mirror::PoolWatcher: 0x56127f7ec5a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8353d0d640 -1 rbd::mirror::PoolWatcher: 0x56127ef30b40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f835a51a640 -1 rbd::mirror::PoolWatcher: 0x56127ef30000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8359d19640 -1 rbd::mirror::PoolWatcher: 0x56127ef30960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127f4da300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T07:11:05.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:05.303+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127f4da300 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-18T07:11:06.163 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:pool 'mirror_parent' does not exist 2026-02-18T07:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-18T07:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirrors cluster1 2026-02-18T07:11:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1 2026-02-18T07:11:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 0 3 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster1:0 '' 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:0 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster1:1 '' 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:1 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster1:2 '' 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:2 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster1:3 '' 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster1:3 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirrors cluster2 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:++ seq 0 3 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster2:0 '' 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:0 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster2:1 '' 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:1 2026-02-18T07:11:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster2:2 '' 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:2 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ stop_mirror cluster2:3 '' 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local cluster=cluster2:3 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ local sig= 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test -n 1 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ return 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' -z 1 ']' 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ test '' = /tmp/tmp.y3X31uzVRt 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ rm -Rf /tmp/tmp.y3X31uzVRt 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ '[' 0 -eq 0 ']' 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ echo OK 2026-02-18T07:11:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial051.stdout:OK 2026-02-18T07:11:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial051.stderr:+ exit 0 2026-02-18T07:11:06.181 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-18T07:11:06.181 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-18T07:11:06.206 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-18T07:11:06.207 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-18T07:11:06.557 DEBUG:teuthology.parallel:result is None 2026-02-18T07:11:06.558 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T07:11:06.585 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T07:11:06.585 DEBUG:teuthology.orchestra.run.trial051:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-18T07:11:06.641 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-18T07:11:06.641 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-18T07:11:06.660 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-18T07:11:06.661 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 10 2026-02-18T07:11:06.661 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-18T07:11:06.661 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-18T07:11:06.678 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-18T07:11:07.924 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-18T07:11:07.925 INFO:teuthology.orchestra.run:waiting for 600 2026-02-18T07:11:07.925 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-18T07:11:07.925 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-18T07:11:07.925 DEBUG:teuthology.orchestra.run.trial051:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-18T07:11:07.928 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-18T07:11:07.928 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 18 2026-02-18T07:11:07.929 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-18T07:11:07.929 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:07.947 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-18T07:11:07.947 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:07.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.945+0000 7ff796cd8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 30164) UID: 1000 2026-02-18T07:11:07.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff781cae640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560d2745ad80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff781cae640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x560d26d28000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4308 object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4308 object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4308 object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff781cae640 -1 rbd::mirror::PoolWatcher: 0x560d277ceb40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff781cae640 -1 rbd::mirror::PoolWatcher: 0x560d27ed9a40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff77bca2640 -1 rbd::mirror::PoolWatcher: 0x560d27ed9680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff77bca2640 -1 rbd::mirror::PoolWatcher: 0x560d27622780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:07.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78f4c9640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78fcca640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78fcca640 -1 rbd::mirror::InstanceWatcher: 0x560d2757a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff795cd6640 -1 rbd::mirror::InstanceWatcher: 0x560d2757ae00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4340 object: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff795cd6640 -1 rbd::mirror::InstanceWatcher: 0x560d2757aa80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4340 object: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7894bd640 -1 rbd::mirror::MirrorStatusUpdater 0x560d271d67e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7894bd640 -1 rbd::mirror::MirrorStatusUpdater 0x560d271d66c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7894bd640 -1 rbd::mirror::MirrorStatusUpdater 0x560d2720c6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d25e46a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.953 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7824af640 -1 rbd::mirror::InstanceWatcher: 0x560d2757ae00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a540 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff781cae640 -1 rbd::mirror::InstanceWatcher: 0x560d2757aa80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78fcca640 -1 rbd::mirror::MirrorStatusUpdater 0x560d271d6d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78fcca640 -1 rbd::mirror::MirrorStatusUpdater 0x560d271d6c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff78fcca640 -1 rbd::mirror::MirrorStatusUpdater 0x560d271d6b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d25e46a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff795cd6640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff795cd6640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a540 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff77b4a1640 -1 rbd::mirror::MirrorStatusUpdater 0x560d2720cea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff77bca2640 -1 rbd::mirror::MirrorStatusUpdater 0x560d2720c7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.954 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff795cd6640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7824af640 -1 rbd::mirror::MirrorStatusUpdater 0x560d2720c360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.946+0000 7ff7964d7640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.947+0000 7ff781cae640 -1 rbd::mirror::MirrorStatusUpdater 0x560d2720c480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.947+0000 7ff795cd6640 -1 rbd::mirror::NamespaceReplayer: 0x560d2719a8c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.947+0000 7ff7964d7640 -1 rbd::mirror::LeaderWatcher: 0x560d274e8000 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.947+0000 7ff7964d7640 -1 rbd::mirror::LeaderWatcher: 0x560d274e8000 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.947+0000 7ff795cd6640 -1 rbd::mirror::LeaderWatcher: 0x560d274e8000 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:07.955 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial051.stderr:2026-02-18T07:11:07.949+0000 7ff795cd6640 -1 rbd::mirror::LeaderWatcher: 0x560d27b26f00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:07.962 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-18T07:11:07.962 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:07.979 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-18T07:11:07.980 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.001+0000 7f9406aac640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 63976) UID: 1000 2026-02-18T07:11:08.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.001+0000 7f9405aaa640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6634 object: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.001+0000 7f94062ab640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6631 object: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.001+0000 7f94062ab640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6631 object: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.001+0000 7f94062ab640 -1 rbd::mirror::InstanceWatcher: 0x55e71a087180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6631 object: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f2283640 -1 rbd::mirror::InstanceWatcher: 0x55e719bda700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f2283640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ff29d640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ffa9e640 -1 rbd::mirror::InstanceWatcher: 0x55e7199c4380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ffa9e640 -1 rbd::mirror::InstanceWatcher: 0x55e71a087180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f1a82640 -1 rbd::mirror::MirrorStatusUpdater 0x55e719c20a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f9405aaa640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998d180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f9405aaa640 -1 rbd::mirror::LeaderWatcher: 0x55e719c85b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f9291640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f9291640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93f9291640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f94062ab640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f94062ab640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f94062ab640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ffa9e640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ffa9e640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f93ffa9e640 -1 rbd::mirror::MirrorStatusUpdater 0x55e71998e240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f9405aaa640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f9405aaa640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.002+0000 7f9405aaa640 -1 rbd::mirror::NamespaceReplayer: 0x55e71998c380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.003+0000 7f9405aaa640 -1 rbd::mirror::LeaderWatcher: 0x55e71963c300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.015 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-18T07:11:08.015 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.030 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-18T07:11:08.030 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff2094ff640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 32310) UID: 1000 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff2024f1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55f8922e8480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff2024f1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55f8922e8fc0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff2024f1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55f8922e8d80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff201cf0640 -1 rbd::mirror::PoolWatcher: 0x55f8918aba40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff201cf0640 -1 rbd::mirror::PoolWatcher: 0x55f8923521e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff201cf0640 -1 rbd::mirror::PoolWatcher: 0x55f892352780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff1fbce4640 -1 rbd::mirror::PoolWatcher: 0x55f892352d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff1fbce4640 -1 rbd::mirror::PoolWatcher: 0x55f892352960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff208cfe640 -1 rbd::mirror::InstanceWatcher: 0x55f891360700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4365 object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff1fb4e3640 -1 rbd::mirror::PoolWatcher: 0x55f88fff9860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-18T07:11:08.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.054+0000 7ff208cfe640 -1 rbd::mirror::InstanceWatcher: 0x55f891154700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4365 object: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1f44d5640 -1 rbd::mirror::InstanceWatcher: 0x55f891360700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1f4cd6640 -1 rbd::mirror::InstanceWatcher: 0x55f891154700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::InstanceWatcher: 0x55f891b0f180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4362 object: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::InstanceWatcher: 0x55f891154000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4362 object: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::InstanceWatcher: 0x55f891154380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4362 object: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1ee4c9640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff208cfe640 -1 rbd::mirror::NamespaceReplayer: 0x55f89127cfc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff201cf0640 -1 rbd::mirror::InstanceWatcher: 0x55f891b0f180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1edcc8640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff201cf0640 -1 rbd::mirror::InstanceWatcher: 0x55f891154380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff201cf0640 -1 rbd::mirror::InstanceWatcher: 0x55f891154000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f89127d180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1f44d5640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891396a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f89127cfc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1f4cd6640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891396b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f89127d180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1fbce4640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff208cfe640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1fb4e3640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff1fbce4640 -1 rbd::mirror::MirrorStatusUpdater 0x55f8911205a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff208cfe640 -1 rbd::mirror::LeaderWatcher: 0x55f8913fbb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2024f1640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff208cfe640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2024f1640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff2084fd640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff201cf0640 -1 rbd::mirror::MirrorStatusUpdater 0x55f891120120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.055+0000 7ff208cfe640 -1 rbd::mirror::NamespaceReplayer: 0x55f890052a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.057+0000 7ff2084fd640 -1 rbd::mirror::LeaderWatcher: 0x55f891e54300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-18T07:11:08.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.057+0000 7ff2084fd640 -1 rbd::mirror::LeaderWatcher: 0x55f891e54300 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-18T07:11:08.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.057+0000 7ff208cfe640 -1 rbd::mirror::LeaderWatcher: 0x55f891e54300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.069 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-18T07:11:08.069 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.080 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-18T07:11:08.080 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ef54b5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 65773) UID: 1000 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ef44b3640 -1 rbd::mirror::InstanceWatcher: 0x5559d38a0000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6828 object: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ef4cb4640 -1 rbd::mirror::InstanceWatcher: 0x5559d410f180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6828 object: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ef4cb4640 -1 rbd::mirror::InstanceWatcher: 0x5559d38a0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6828 object: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ee048b640 -1 rbd::mirror::InstanceWatcher: 0x5559d3dd2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0ee0c8c640 -1 rbd::mirror::InstanceWatcher: 0x5559d3dd2e00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0eedca6640 -1 rbd::mirror::InstanceWatcher: 0x5559d38a0000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0eee4a7640 -1 rbd::mirror::InstanceWatcher: 0x5559d410f180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.109+0000 7f0eee4a7640 -1 rbd::mirror::InstanceWatcher: 0x5559d38a0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ee7c9a640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef4cb4640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ee7c9a640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ee7c9a640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef44b3640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0eedca6640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef44b3640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef44b3640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0eee4a7640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0eee4a7640 -1 rbd::mirror::MirrorStatusUpdater 0x5559d3a2c120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef44b3640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.110+0000 7f0ef44b3640 -1 rbd::mirror::NamespaceReplayer: 0x5559d3a2a1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.113 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.111+0000 7f0ef44b3640 -1 rbd::mirror::LeaderWatcher: 0x5559d37e8300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.121 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-18T07:11:08.121 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.133 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-18T07:11:08.133 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.162 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8361528640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 35227) UID: 1000 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8360d27640 -1 rbd::mirror::InstanceWatcher: 0x56127e80b880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4497 object: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8360526640 -1 rbd::mirror::InstanceWatcher: 0x56127f197180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4497 object: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8360526640 -1 rbd::mirror::InstanceWatcher: 0x56127e80a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4497 object: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8360526640 -1 rbd::mirror::InstanceWatcher: 0x56127e80bc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4499 object: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8360526640 -1 rbd::mirror::InstanceWatcher: 0x56127e9d8700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4499 object: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.161+0000 7f8359d19640 -1 rbd::mirror::InstanceWatcher: 0x56127e80b880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f835a51a640 -1 rbd::mirror::InstanceWatcher: 0x56127f197180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.163 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f834c4fe640 -1 rbd::mirror::InstanceWatcher: 0x56127e80bc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f834ccff640 -1 rbd::mirror::InstanceWatcher: 0x56127e9d8700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8359d19640 -1 rbd::mirror::InstanceWatcher: 0x56127e80a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f835350c640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f835350c640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8345cf1640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360d27640 -1 rbd::mirror::NamespaceReplayer: 0x56127e8f8fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f834c4fe640 -1 rbd::mirror::MirrorStatusUpdater 0x56127ea1cb40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360d27640 -1 rbd::mirror::NamespaceReplayer: 0x56127e8f9180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8353d0d640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b65a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360d27640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f835a51a640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f834ccff640 -1 rbd::mirror::MirrorStatusUpdater 0x56127ea1ca20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.164 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127e8f8fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8359d19640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f835a51a640 -1 rbd::mirror::MirrorStatusUpdater 0x56127e7b6240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360526640 -1 rbd::mirror::NamespaceReplayer: 0x56127d6d6e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.165 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.162+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127ea83b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.166 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial051.stderr:2026-02-18T07:11:08.164+0000 7f8360d27640 -1 rbd::mirror::LeaderWatcher: 0x56127f4da300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-18T07:11:08.177 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.190 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-18T07:11:08.190 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.219 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial051.stderr:2026-02-18T07:11:08.217+0000 7fec8b6e2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 68035) UID: 1000 2026-02-18T07:11:08.225 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-18T07:11:08.225 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.235 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-18T07:11:08.235 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.264+0000 7ff75a868640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 66157) UID: 1000 2026-02-18T07:11:08.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.264+0000 7ff759866640 -1 rbd::mirror::InstanceWatcher: 0x5569ecbcae00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5986 object: (2) No such file or directory 2026-02-18T07:11:08.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.264+0000 7ff75a067640 -1 rbd::mirror::InstanceWatcher: 0x5569ecbca380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5986 object: (2) No such file or directory 2026-02-18T07:11:08.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.264+0000 7ff75a067640 -1 rbd::mirror::InstanceWatcher: 0x5569ed0f5180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-18T07:11:08.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.264+0000 7ff759866640 -1 rbd::mirror::InstanceWatcher: 0x5569ed4e2000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::InstanceWatcher: 0x5569ed4e3c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5984 object: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74583e640 -1 rbd::mirror::InstanceWatcher: 0x5569ecbcae00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff753059640 -1 rbd::mirror::InstanceWatcher: 0x5569ed0f5180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75385a640 -1 rbd::mirror::InstanceWatcher: 0x5569ed4e2000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74603f640 -1 rbd::mirror::InstanceWatcher: 0x5569ecbca380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff753059640 -1 rbd::mirror::InstanceWatcher: 0x5569ed4e3c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff73f832640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec989320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.267 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff759866640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70f180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74c84c640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec7125a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74d04d640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec712360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff759866640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff73f031640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec988d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70f340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74c84c640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec712480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74583e640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec988a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff759866640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70f180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75385a640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec712000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff753059640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec712120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff74603f640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec988b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70f340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75385a640 -1 rbd::mirror::MirrorStatusUpdater 0x5569ec712240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff759866640 -1 rbd::mirror::NamespaceReplayer: 0x5569ec70e380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.265+0000 7ff75a067640 -1 rbd::mirror::LeaderWatcher: 0x5569eca0bb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.268 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial051.stderr:2026-02-18T07:11:08.267+0000 7ff75a067640 -1 rbd::mirror::LeaderWatcher: 0x5569ed432300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.277 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-18T07:11:08.277 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-18T07:11:08.288 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-18T07:11:08.288 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:08.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4f7e3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 58929) UID: 1000 2026-02-18T07:11:08.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4e7e1640 -1 rbd::mirror::InstanceWatcher: 0x55650f056700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5574 object: (2) No such file or directory 2026-02-18T07:11:08.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4efe2640 -1 rbd::mirror::InstanceWatcher: 0x55650f4dd180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5571 object: (2) No such file or directory 2026-02-18T07:11:08.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4efe2640 -1 rbd::mirror::InstanceWatcher: 0x55650f056a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5574 object: (2) No such file or directory 2026-02-18T07:11:08.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4e7e1640 -1 rbd::mirror::InstanceWatcher: 0x55650edae000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5571 object: (2) No such file or directory 2026-02-18T07:11:08.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a4e7e1640 -1 rbd::mirror::InstanceWatcher: 0x55650edafc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5571 object: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a3a7b9640 -1 rbd::mirror::InstanceWatcher: 0x55650f056700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a487d5640 -1 rbd::mirror::InstanceWatcher: 0x55650f4dd180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.316+0000 7f2a3afba640 -1 rbd::mirror::InstanceWatcher: 0x55650f056a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a347ad640 -1 rbd::mirror::MirrorStatusUpdater 0x55650f087320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650ef26380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a47fd4640 -1 rbd::mirror::InstanceWatcher: 0x55650edae000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a487d5640 -1 rbd::mirror::InstanceWatcher: 0x55650edafc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a3a7b9640 -1 rbd::mirror::MirrorStatusUpdater 0x55650f086b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650ef26540 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a417c7640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee025a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a3afba640 -1 rbd::mirror::MirrorStatusUpdater 0x55650f086a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650ef26380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a41fc8640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee02360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a41fc8640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee02480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4efe2640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4efe2640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a47fd4640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee02000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a487d5640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee02120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4e7e1640 -1 rbd::mirror::LeaderWatcher: 0x55650f12c900 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a47fd4640 -1 rbd::mirror::MirrorStatusUpdater 0x55650ee02240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-18T07:11:08.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.317+0000 7f2a4efe2640 -1 rbd::mirror::NamespaceReplayer: 0x55650da32a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-18T07:11:08.320 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial051.stderr:2026-02-18T07:11:08.319+0000 7f2a4e7e1640 -1 rbd::mirror::LeaderWatcher: 0x55650ebce300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-18T07:11:08.330 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-18T07:11:08.330 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-18T07:11:08.341 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-18T07:11:08.341 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T07:11:08.551 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:08.551 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T07:11:08.560 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":1458,"stamp":"2026-02-18T07:11:08.121790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2904596,"kb_used_data":9444,"kb_used_omap":2635,"kb_used_meta":2892468,"kb_avail":280210924,"statfs":{"total":289910292480,"available":286935986176,"internally_reserved":0,"allocated":9670656,"data_stored":6299993,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":2699160,"internal_metadata":2961887336},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536862774,"num_objects":-173,"num_object_clones":0,"num_object_copies":-346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-173,"num_whiteouts":0,"num_read":-114341,"num_read_kb":-39619297,"num_write":-1201557,"num_write_kb":-452612431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-2,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001005"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570063+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T07:11:06.570063+0000","last_peered":"2026-02-18T07:11:06.570063+0000","last_clean":"2026-02-18T07:11:06.570063+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T07:11:06.570063+0000","last_undegraded":"2026-02-18T07:11:06.570063+0000","last_fullsized":"2026-02-18T07:11:06.570063+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569482+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T07:11:06.569482+0000","last_peered":"2026-02-18T07:11:06.569482+0000","last_clean":"2026-02-18T07:11:06.569482+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T07:11:06.569482+0000","last_undegraded":"2026-02-18T07:11:06.569482+0000","last_fullsized":"2026-02-18T07:11:06.569482+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569839+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T07:11:06.569839+0000","last_peered":"2026-02-18T07:11:06.569839+0000","last_clean":"2026-02-18T07:11:06.569839+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T07:11:06.569839+0000","last_undegraded":"2026-02-18T07:11:06.569839+0000","last_fullsized":"2026-02-18T07:11:06.569839+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.567695+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T07:11:06.567695+0000","last_peered":"2026-02-18T07:11:06.567695+0000","last_clean":"2026-02-18T07:11:06.567695+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T07:11:06.567695+0000","last_undegraded":"2026-02-18T07:11:06.567695+0000","last_fullsized":"2026-02-18T07:11:06.567695+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":783,"reported_epoch":386,"state":"active+clean","last_fresh":"2026-02-18T07:10:35.083843+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T07:10:35.083843+0000","last_peered":"2026-02-18T07:10:35.083843+0000","last_clean":"2026-02-18T07:10:35.083843+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T07:10:35.083843+0000","last_undegraded":"2026-02-18T07:10:35.083843+0000","last_fullsized":"2026-02-18T07:10:35.083843+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.100355+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T07:11:04.100355+0000","last_peered":"2026-02-18T07:11:04.100355+0000","last_clean":"2026-02-18T07:11:04.100355+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T07:11:04.100355+0000","last_undegraded":"2026-02-18T07:11:04.100355+0000","last_fullsized":"2026-02-18T07:11:04.100355+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.097223+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T07:11:04.097223+0000","last_peered":"2026-02-18T07:11:04.097223+0000","last_clean":"2026-02-18T07:11:04.097223+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T07:11:04.097223+0000","last_undegraded":"2026-02-18T07:11:04.097223+0000","last_fullsized":"2026-02-18T07:11:04.097223+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":785,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570054+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T07:11:06.570054+0000","last_peered":"2026-02-18T07:11:06.570054+0000","last_clean":"2026-02-18T07:11:06.570054+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T07:11:06.570054+0000","last_undegraded":"2026-02-18T07:11:06.570054+0000","last_fullsized":"2026-02-18T07:11:06.570054+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":873,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569384+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T07:11:06.569384+0000","last_peered":"2026-02-18T07:11:06.569384+0000","last_clean":"2026-02-18T07:11:06.569384+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T07:11:06.569384+0000","last_undegraded":"2026-02-18T07:11:06.569384+0000","last_fullsized":"2026-02-18T07:11:06.569384+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738799,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":623992,"kb_used_data":3112,"kb_used_omap":1079,"kb_used_meta":619784,"kb_avail":93747848,"statfs":{"total":96636764160,"available":95997796352,"internally_reserved":0,"allocated":3186688,"data_stored":1707776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1105384,"internal_metadata":634659352},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738800,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":949748,"kb_used_data":3172,"kb_used_omap":377,"kb_used_meta":946182,"kb_avail":93422092,"statfs":{"total":96636764160,"available":95664222208,"internally_reserved":0,"allocated":3248128,"data_stored":2298163,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":386968,"internal_metadata":968890472},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738800,"num_pgs":93,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1330856,"kb_used_data":3160,"kb_used_omap":1178,"kb_used_meta":1326501,"kb_avail":93040984,"statfs":{"total":96636764160,"available":95273967616,"internally_reserved":0,"allocated":3235840,"data_stored":2294054,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1206808,"internal_metadata":1358337512},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:08.560 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T07:11:08.741 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:08.742 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T07:11:08.750 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":1458,"stamp":"2026-02-18T07:11:08.121790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2904596,"kb_used_data":9444,"kb_used_omap":2635,"kb_used_meta":2892468,"kb_avail":280210924,"statfs":{"total":289910292480,"available":286935986176,"internally_reserved":0,"allocated":9670656,"data_stored":6299993,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":2699160,"internal_metadata":2961887336},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536862774,"num_objects":-173,"num_object_clones":0,"num_object_copies":-346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-173,"num_whiteouts":0,"num_read":-114341,"num_read_kb":-39619297,"num_write":-1201557,"num_write_kb":-452612431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-2,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001005"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570063+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T07:11:06.570063+0000","last_peered":"2026-02-18T07:11:06.570063+0000","last_clean":"2026-02-18T07:11:06.570063+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T07:11:06.570063+0000","last_undegraded":"2026-02-18T07:11:06.570063+0000","last_fullsized":"2026-02-18T07:11:06.570063+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569482+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T07:11:06.569482+0000","last_peered":"2026-02-18T07:11:06.569482+0000","last_clean":"2026-02-18T07:11:06.569482+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T07:11:06.569482+0000","last_undegraded":"2026-02-18T07:11:06.569482+0000","last_fullsized":"2026-02-18T07:11:06.569482+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569839+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T07:11:06.569839+0000","last_peered":"2026-02-18T07:11:06.569839+0000","last_clean":"2026-02-18T07:11:06.569839+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T07:11:06.569839+0000","last_undegraded":"2026-02-18T07:11:06.569839+0000","last_fullsized":"2026-02-18T07:11:06.569839+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.567695+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T07:11:06.567695+0000","last_peered":"2026-02-18T07:11:06.567695+0000","last_clean":"2026-02-18T07:11:06.567695+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T07:11:06.567695+0000","last_undegraded":"2026-02-18T07:11:06.567695+0000","last_fullsized":"2026-02-18T07:11:06.567695+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":783,"reported_epoch":386,"state":"active+clean","last_fresh":"2026-02-18T07:10:35.083843+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T07:10:35.083843+0000","last_peered":"2026-02-18T07:10:35.083843+0000","last_clean":"2026-02-18T07:10:35.083843+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T07:10:35.083843+0000","last_undegraded":"2026-02-18T07:10:35.083843+0000","last_fullsized":"2026-02-18T07:10:35.083843+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.100355+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T07:11:04.100355+0000","last_peered":"2026-02-18T07:11:04.100355+0000","last_clean":"2026-02-18T07:11:04.100355+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T07:11:04.100355+0000","last_undegraded":"2026-02-18T07:11:04.100355+0000","last_fullsized":"2026-02-18T07:11:04.100355+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.097223+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T07:11:04.097223+0000","last_peered":"2026-02-18T07:11:04.097223+0000","last_clean":"2026-02-18T07:11:04.097223+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T07:11:04.097223+0000","last_undegraded":"2026-02-18T07:11:04.097223+0000","last_fullsized":"2026-02-18T07:11:04.097223+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":785,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570054+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T07:11:06.570054+0000","last_peered":"2026-02-18T07:11:06.570054+0000","last_clean":"2026-02-18T07:11:06.570054+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T07:11:06.570054+0000","last_undegraded":"2026-02-18T07:11:06.570054+0000","last_fullsized":"2026-02-18T07:11:06.570054+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":873,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569384+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T07:11:06.569384+0000","last_peered":"2026-02-18T07:11:06.569384+0000","last_clean":"2026-02-18T07:11:06.569384+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T07:11:06.569384+0000","last_undegraded":"2026-02-18T07:11:06.569384+0000","last_fullsized":"2026-02-18T07:11:06.569384+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738799,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":623992,"kb_used_data":3112,"kb_used_omap":1079,"kb_used_meta":619784,"kb_avail":93747848,"statfs":{"total":96636764160,"available":95997796352,"internally_reserved":0,"allocated":3186688,"data_stored":1707776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1105384,"internal_metadata":634659352},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738800,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":949748,"kb_used_data":3172,"kb_used_omap":377,"kb_used_meta":946182,"kb_avail":93422092,"statfs":{"total":96636764160,"available":95664222208,"internally_reserved":0,"allocated":3248128,"data_stored":2298163,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":386968,"internal_metadata":968890472},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738800,"num_pgs":93,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1330856,"kb_used_data":3160,"kb_used_omap":1178,"kb_used_meta":1326501,"kb_avail":93040984,"statfs":{"total":96636764160,"available":95273967616,"internally_reserved":0,"allocated":3235840,"data_stored":2294054,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1206808,"internal_metadata":1358337512},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:08.750 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-18T07:11:08.751 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T07:11:08.932 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:08.932 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T07:11:08.940 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":1458,"stamp":"2026-02-18T07:11:08.121790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2904596,"kb_used_data":9444,"kb_used_omap":2635,"kb_used_meta":2892468,"kb_avail":280210924,"statfs":{"total":289910292480,"available":286935986176,"internally_reserved":0,"allocated":9670656,"data_stored":6299993,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":2699160,"internal_metadata":2961887336},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536862774,"num_objects":-173,"num_object_clones":0,"num_object_copies":-346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-173,"num_whiteouts":0,"num_read":-114341,"num_read_kb":-39619297,"num_write":-1201557,"num_write_kb":-452612431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-2,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001005"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570063+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T07:11:06.570063+0000","last_peered":"2026-02-18T07:11:06.570063+0000","last_clean":"2026-02-18T07:11:06.570063+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T07:11:06.570063+0000","last_undegraded":"2026-02-18T07:11:06.570063+0000","last_fullsized":"2026-02-18T07:11:06.570063+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569482+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T07:11:06.569482+0000","last_peered":"2026-02-18T07:11:06.569482+0000","last_clean":"2026-02-18T07:11:06.569482+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T07:11:06.569482+0000","last_undegraded":"2026-02-18T07:11:06.569482+0000","last_fullsized":"2026-02-18T07:11:06.569482+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569839+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T07:11:06.569839+0000","last_peered":"2026-02-18T07:11:06.569839+0000","last_clean":"2026-02-18T07:11:06.569839+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T07:11:06.569839+0000","last_undegraded":"2026-02-18T07:11:06.569839+0000","last_fullsized":"2026-02-18T07:11:06.569839+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.567695+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T07:11:06.567695+0000","last_peered":"2026-02-18T07:11:06.567695+0000","last_clean":"2026-02-18T07:11:06.567695+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T07:11:06.567695+0000","last_undegraded":"2026-02-18T07:11:06.567695+0000","last_fullsized":"2026-02-18T07:11:06.567695+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":783,"reported_epoch":386,"state":"active+clean","last_fresh":"2026-02-18T07:10:35.083843+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T07:10:35.083843+0000","last_peered":"2026-02-18T07:10:35.083843+0000","last_clean":"2026-02-18T07:10:35.083843+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T07:10:35.083843+0000","last_undegraded":"2026-02-18T07:10:35.083843+0000","last_fullsized":"2026-02-18T07:10:35.083843+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.100355+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T07:11:04.100355+0000","last_peered":"2026-02-18T07:11:04.100355+0000","last_clean":"2026-02-18T07:11:04.100355+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T07:11:04.100355+0000","last_undegraded":"2026-02-18T07:11:04.100355+0000","last_fullsized":"2026-02-18T07:11:04.100355+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.097223+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T07:11:04.097223+0000","last_peered":"2026-02-18T07:11:04.097223+0000","last_clean":"2026-02-18T07:11:04.097223+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T07:11:04.097223+0000","last_undegraded":"2026-02-18T07:11:04.097223+0000","last_fullsized":"2026-02-18T07:11:04.097223+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":785,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570054+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T07:11:06.570054+0000","last_peered":"2026-02-18T07:11:06.570054+0000","last_clean":"2026-02-18T07:11:06.570054+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T07:11:06.570054+0000","last_undegraded":"2026-02-18T07:11:06.570054+0000","last_fullsized":"2026-02-18T07:11:06.570054+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":873,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569384+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T07:11:06.569384+0000","last_peered":"2026-02-18T07:11:06.569384+0000","last_clean":"2026-02-18T07:11:06.569384+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T07:11:06.569384+0000","last_undegraded":"2026-02-18T07:11:06.569384+0000","last_fullsized":"2026-02-18T07:11:06.569384+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738799,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":623992,"kb_used_data":3112,"kb_used_omap":1079,"kb_used_meta":619784,"kb_avail":93747848,"statfs":{"total":96636764160,"available":95997796352,"internally_reserved":0,"allocated":3186688,"data_stored":1707776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1105384,"internal_metadata":634659352},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738800,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":949748,"kb_used_data":3172,"kb_used_omap":377,"kb_used_meta":946182,"kb_avail":93422092,"statfs":{"total":96636764160,"available":95664222208,"internally_reserved":0,"allocated":3248128,"data_stored":2298163,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":386968,"internal_metadata":968890472},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738800,"num_pgs":93,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1330856,"kb_used_data":3160,"kb_used_omap":1178,"kb_used_meta":1326501,"kb_avail":93040984,"statfs":{"total":96636764160,"available":95273967616,"internally_reserved":0,"allocated":3235840,"data_stored":2294054,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1206808,"internal_metadata":1358337512},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:08.941 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-18T07:11:09.121 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:09.121 INFO:teuthology.orchestra.run.trial051.stdout:{"epoch":388,"fsid":"9115e2ce-4d18-4172-bcd4-e2000f709e06","created":"2026-02-18T06:35:19.917963+0000","modified":"2026-02-18T07:11:06.100421+0000","last_up_change":"2026-02-18T06:35:23.923562+0000","last_in_change":"2026-02-18T06:35:20.597066+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:25.821503+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:27.797884+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e9c2b530-d1be-41b5-9131-32932905caee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6816","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6817","nonce":3495423820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6818","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6819","nonce":3495423820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6822","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6823","nonce":3495423820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6820","nonce":3495423820},{"type":"v1","addr":"10.20.193.51:6821","nonce":3495423820}]},"public_addr":"10.20.193.51:6817/3495423820","cluster_addr":"10.20.193.51:6819/3495423820","heartbeat_back_addr":"10.20.193.51:6823/3495423820","heartbeat_front_addr":"10.20.193.51:6821/3495423820","state":["exists","up"]},{"osd":1,"uuid":"db1cbb05-e951-4302-b391-ec61dd00ce9b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6808","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6809","nonce":1672938220}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6810","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6811","nonce":1672938220}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6814","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6815","nonce":1672938220}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6812","nonce":1672938220},{"type":"v1","addr":"10.20.193.51:6813","nonce":1672938220}]},"public_addr":"10.20.193.51:6809/1672938220","cluster_addr":"10.20.193.51:6811/1672938220","heartbeat_back_addr":"10.20.193.51:6815/1672938220","heartbeat_front_addr":"10.20.193.51:6813/1672938220","state":["exists","up"]},{"osd":2,"uuid":"fe301036-a947-4204-87a7-3efc38575b5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6800","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6801","nonce":3566361000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6802","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6803","nonce":3566361000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6806","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6807","nonce":3566361000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.51:6804","nonce":3566361000},{"type":"v1","addr":"10.20.193.51:6805","nonce":3566361000}]},"public_addr":"10.20.193.51:6801/3566361000","cluster_addr":"10.20.193.51:6803/3566361000","heartbeat_back_addr":"10.20.193.51:6807/3566361000","heartbeat_front_addr":"10.20.193.51:6805/3566361000","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:22.477374+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:22.371924+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:22.385408+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T07:11:10.131 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-18T07:11:10.131 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:10.215 INFO:teuthology.orchestra.run.trial051.stdout:{ 2026-02-18T07:11:10.215 INFO:teuthology.orchestra.run.trial051.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:10.215 INFO:teuthology.orchestra.run.trial051.stdout:} 2026-02-18T07:11:10.221 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-18T07:11:10.406 INFO:teuthology.orchestra.run.trial051.stderr:instructed osd(s) 0 to deep-scrub 2026-02-18T07:11:10.416 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-18T07:11:10.416 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:10.498 INFO:teuthology.orchestra.run.trial051.stdout:{ 2026-02-18T07:11:10.498 INFO:teuthology.orchestra.run.trial051.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:10.498 INFO:teuthology.orchestra.run.trial051.stdout:} 2026-02-18T07:11:10.504 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-18T07:11:10.685 INFO:teuthology.orchestra.run.trial051.stderr:instructed osd(s) 1 to deep-scrub 2026-02-18T07:11:10.695 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-18T07:11:10.695 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:10.780 INFO:teuthology.orchestra.run.trial051.stdout:{ 2026-02-18T07:11:10.781 INFO:teuthology.orchestra.run.trial051.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:10.781 INFO:teuthology.orchestra.run.trial051.stdout:} 2026-02-18T07:11:10.787 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-18T07:11:10.972 INFO:teuthology.orchestra.run.trial051.stderr:instructed osd(s) 2 to deep-scrub 2026-02-18T07:11:10.984 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T07:11:11.164 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:11.165 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T07:11:11.174 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":1459,"stamp":"2026-02-18T07:11:10.121975+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":187,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2922664,"kb_used_data":9464,"kb_used_omap":2636,"kb_used_meta":2910515,"kb_avail":280192856,"statfs":{"total":289910292480,"available":286917484544,"internally_reserved":0,"allocated":9691136,"data_stored":6307715,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":2699644,"internal_metadata":2980368004},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536862774,"num_objects":-173,"num_object_clones":0,"num_object_copies":-346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-173,"num_whiteouts":0,"num_read":-114354,"num_read_kb":-39619310,"num_write":-1201560,"num_write_kb":-452612434,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-2,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000903"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570063+0000","last_change":"2026-02-18T06:35:30.940594+0000","last_active":"2026-02-18T07:11:06.570063+0000","last_peered":"2026-02-18T07:11:06.570063+0000","last_clean":"2026-02-18T07:11:06.570063+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T07:11:06.570063+0000","last_undegraded":"2026-02-18T07:11:06.570063+0000","last_fullsized":"2026-02-18T07:11:06.570063+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:51:38.922915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":783,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569482+0000","last_change":"2026-02-18T06:35:30.940018+0000","last_active":"2026-02-18T07:11:06.569482+0000","last_peered":"2026-02-18T07:11:06.569482+0000","last_clean":"2026-02-18T07:11:06.569482+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T07:11:06.569482+0000","last_undegraded":"2026-02-18T07:11:06.569482+0000","last_fullsized":"2026-02-18T07:11:06.569482+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:05:03.343514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569839+0000","last_change":"2026-02-18T06:35:30.940337+0000","last_active":"2026-02-18T07:11:06.569839+0000","last_peered":"2026-02-18T07:11:06.569839+0000","last_clean":"2026-02-18T07:11:06.569839+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T07:11:06.569839+0000","last_undegraded":"2026-02-18T07:11:06.569839+0000","last_fullsized":"2026-02-18T07:11:06.569839+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:21:43.513943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":784,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.567695+0000","last_change":"2026-02-18T06:35:30.939996+0000","last_active":"2026-02-18T07:11:06.567695+0000","last_peered":"2026-02-18T07:11:06.567695+0000","last_clean":"2026-02-18T07:11:06.567695+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T07:11:06.567695+0000","last_undegraded":"2026-02-18T07:11:06.567695+0000","last_fullsized":"2026-02-18T07:11:06.567695+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:31:30.608888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":787,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.105037+0000","last_change":"2026-02-18T06:35:30.941644+0000","last_active":"2026-02-18T07:11:06.105037+0000","last_peered":"2026-02-18T07:11:06.105037+0000","last_clean":"2026-02-18T07:11:06.105037+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T07:11:06.105037+0000","last_undegraded":"2026-02-18T07:11:06.105037+0000","last_fullsized":"2026-02-18T07:11:06.105037+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:49:55.767961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.100355+0000","last_change":"2026-02-18T06:35:31.440845+0000","last_active":"2026-02-18T07:11:04.100355+0000","last_peered":"2026-02-18T07:11:04.100355+0000","last_clean":"2026-02-18T07:11:04.100355+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T07:11:04.100355+0000","last_undegraded":"2026-02-18T07:11:04.100355+0000","last_fullsized":"2026-02-18T07:11:04.100355+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:14:37.166594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":783,"reported_epoch":387,"state":"active+clean","last_fresh":"2026-02-18T07:11:04.097223+0000","last_change":"2026-02-18T06:35:31.440775+0000","last_active":"2026-02-18T07:11:04.097223+0000","last_peered":"2026-02-18T07:11:04.097223+0000","last_clean":"2026-02-18T07:11:04.097223+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T07:11:04.097223+0000","last_undegraded":"2026-02-18T07:11:04.097223+0000","last_fullsized":"2026-02-18T07:11:04.097223+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:26:58.611444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":785,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.570054+0000","last_change":"2026-02-18T06:35:30.940315+0000","last_active":"2026-02-18T07:11:06.570054+0000","last_peered":"2026-02-18T07:11:06.570054+0000","last_clean":"2026-02-18T07:11:06.570054+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T07:11:06.570054+0000","last_undegraded":"2026-02-18T07:11:06.570054+0000","last_fullsized":"2026-02-18T07:11:06.570054+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:27.932403+0000","last_clean_scrub_stamp":"2026-02-18T06:35:27.932403+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:29:43.275806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":873,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.569384+0000","last_change":"2026-02-18T06:35:27.375416+0000","last_active":"2026-02-18T07:11:06.569384+0000","last_peered":"2026-02-18T07:11:06.569384+0000","last_clean":"2026-02-18T07:11:06.569384+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T07:11:06.569384+0000","last_undegraded":"2026-02-18T07:11:06.569384+0000","last_fullsized":"2026-02-18T07:11:06.569384+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:25.929461+0000","last_clean_scrub_stamp":"2026-02-18T06:35:25.929461+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:52:06.278635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738799,"num_pgs":83,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":623992,"kb_used_data":3112,"kb_used_omap":1079,"kb_used_meta":619784,"kb_avail":93747848,"statfs":{"total":96636764160,"available":95997796352,"internally_reserved":0,"allocated":3186688,"data_stored":1707776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1105384,"internal_metadata":634659352},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738800,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":949748,"kb_used_data":3172,"kb_used_omap":377,"kb_used_meta":946182,"kb_avail":93422092,"statfs":{"total":96636764160,"available":95664222208,"internally_reserved":0,"allocated":3248128,"data_stored":2298163,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":386968,"internal_metadata":968890472},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738801,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1348924,"kb_used_data":3180,"kb_used_omap":1178,"kb_used_meta":1344549,"kb_avail":93022916,"statfs":{"total":96636764160,"available":95255465984,"internally_reserved":0,"allocated":3256320,"data_stored":2301776,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1207292,"internal_metadata":1376818180},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:11.182 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.182 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.182 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.182 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.183 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.183 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.183 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.184 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-18T06:35:27.932403+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=27, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.184 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-18T06:35:25.929461+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=25, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=9, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:11.184 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-18T07:11:31.185 DEBUG:teuthology.orchestra.run.trial051:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-18T07:11:31.376 INFO:teuthology.orchestra.run.trial051.stdout: 2026-02-18T07:11:31.377 INFO:teuthology.orchestra.run.trial051.stderr:dumped all 2026-02-18T07:11:31.386 INFO:teuthology.orchestra.run.trial051.stdout:{"pg_ready":true,"pg_map":{"version":1469,"stamp":"2026-02-18T07:11:30.123604+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":2922680,"kb_used_data":9480,"kb_used_omap":2639,"kb_used_meta":2910512,"kb_avail":280192840,"statfs":{"total":289910292480,"available":286917468160,"internally_reserved":0,"allocated":9707520,"data_stored":6314941,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":2703004,"internal_metadata":2980364644},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000976"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":791,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:15.765593+0000","last_change":"2026-02-18T07:11:15.765593+0000","last_active":"2026-02-18T07:11:15.765593+0000","last_peered":"2026-02-18T07:11:15.765593+0000","last_clean":"2026-02-18T07:11:15.765593+0000","last_became_active":"2026-02-18T06:35:28.939437+0000","last_became_peered":"2026-02-18T06:35:28.939437+0000","last_unstale":"2026-02-18T07:11:15.765593+0000","last_undegraded":"2026-02-18T07:11:15.765593+0000","last_fullsized":"2026-02-18T07:11:15.765593+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:15.765536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:15.765536+0000","last_clean_scrub_stamp":"2026-02-18T07:11:15.765536+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T15:41:54.959631+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00085399,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":791,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:14.755806+0000","last_change":"2026-02-18T07:11:14.755806+0000","last_active":"2026-02-18T07:11:14.755806+0000","last_peered":"2026-02-18T07:11:14.755806+0000","last_clean":"2026-02-18T07:11:14.755806+0000","last_became_active":"2026-02-18T06:35:28.939316+0000","last_became_peered":"2026-02-18T06:35:28.939316+0000","last_unstale":"2026-02-18T07:11:14.755806+0000","last_undegraded":"2026-02-18T07:11:14.755806+0000","last_fullsized":"2026-02-18T07:11:14.755806+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:14.755736+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:14.755736+0000","last_clean_scrub_stamp":"2026-02-18T07:11:14.755736+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:33:08.020283+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000389701,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":792,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:13.746999+0000","last_change":"2026-02-18T07:11:13.746999+0000","last_active":"2026-02-18T07:11:13.746999+0000","last_peered":"2026-02-18T07:11:13.746999+0000","last_clean":"2026-02-18T07:11:13.746999+0000","last_became_active":"2026-02-18T06:35:28.939414+0000","last_became_peered":"2026-02-18T06:35:28.939414+0000","last_unstale":"2026-02-18T07:11:13.746999+0000","last_undegraded":"2026-02-18T07:11:13.746999+0000","last_fullsized":"2026-02-18T07:11:13.746999+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:13.746941+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:13.746941+0000","last_clean_scrub_stamp":"2026-02-18T07:11:13.746941+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:34:22.167197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062713000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":792,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:11.735471+0000","last_change":"2026-02-18T07:11:11.735471+0000","last_active":"2026-02-18T07:11:11.735471+0000","last_peered":"2026-02-18T07:11:11.735471+0000","last_clean":"2026-02-18T07:11:11.735471+0000","last_became_active":"2026-02-18T06:35:28.938932+0000","last_became_peered":"2026-02-18T06:35:28.938932+0000","last_unstale":"2026-02-18T07:11:11.735471+0000","last_undegraded":"2026-02-18T07:11:11.735471+0000","last_fullsized":"2026-02-18T07:11:11.735471+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:11.735384+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:11.735384+0000","last_clean_scrub_stamp":"2026-02-18T07:11:11.735384+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:25:28.278048+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000353333,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"388'3","reported_seq":796,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:10.477916+0000","last_change":"2026-02-18T07:11:10.477916+0000","last_active":"2026-02-18T07:11:10.477916+0000","last_peered":"2026-02-18T07:11:10.477916+0000","last_clean":"2026-02-18T07:11:10.477916+0000","last_became_active":"2026-02-18T06:35:28.939614+0000","last_became_peered":"2026-02-18T06:35:28.939614+0000","last_unstale":"2026-02-18T07:11:10.477916+0000","last_undegraded":"2026-02-18T07:11:10.477916+0000","last_fullsized":"2026-02-18T07:11:10.477916+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'3","last_scrub_stamp":"2026-02-18T07:11:10.477882+0000","last_deep_scrub":"388'3","last_deep_scrub_stamp":"2026-02-18T07:11:10.477882+0000","last_clean_scrub_stamp":"2026-02-18T07:11:10.477882+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:11:06.534115+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00066758500000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":793,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:11.952002+0000","last_change":"2026-02-18T07:11:11.952002+0000","last_active":"2026-02-18T07:11:11.952002+0000","last_peered":"2026-02-18T07:11:11.952002+0000","last_clean":"2026-02-18T07:11:11.952002+0000","last_became_active":"2026-02-18T06:35:28.940123+0000","last_became_peered":"2026-02-18T06:35:28.940123+0000","last_unstale":"2026-02-18T07:11:11.952002+0000","last_undegraded":"2026-02-18T07:11:11.952002+0000","last_fullsized":"2026-02-18T07:11:11.952002+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:11.951950+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:11.951950+0000","last_clean_scrub_stamp":"2026-02-18T07:11:11.951950+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T12:24:10.973717+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00025150500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":793,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:12.955111+0000","last_change":"2026-02-18T07:11:12.955111+0000","last_active":"2026-02-18T07:11:12.955111+0000","last_peered":"2026-02-18T07:11:12.955111+0000","last_clean":"2026-02-18T07:11:12.955111+0000","last_became_active":"2026-02-18T06:35:28.939321+0000","last_became_peered":"2026-02-18T06:35:28.939321+0000","last_unstale":"2026-02-18T07:11:12.955111+0000","last_undegraded":"2026-02-18T07:11:12.955111+0000","last_fullsized":"2026-02-18T07:11:12.955111+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:12.955053+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:12.955053+0000","last_clean_scrub_stamp":"2026-02-18T07:11:12.955053+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:32:16.464353+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00022174,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"388'2","reported_seq":794,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:10.766985+0000","last_change":"2026-02-18T07:11:10.766985+0000","last_active":"2026-02-18T07:11:10.766985+0000","last_peered":"2026-02-18T07:11:10.766985+0000","last_clean":"2026-02-18T07:11:10.766985+0000","last_became_active":"2026-02-18T06:35:28.939044+0000","last_became_peered":"2026-02-18T06:35:28.939044+0000","last_unstale":"2026-02-18T07:11:10.766985+0000","last_undegraded":"2026-02-18T07:11:10.766985+0000","last_fullsized":"2026-02-18T07:11:10.766985+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'2","last_scrub_stamp":"2026-02-18T07:11:10.766943+0000","last_deep_scrub":"388'2","last_deep_scrub_stamp":"2026-02-18T07:11:10.766943+0000","last_clean_scrub_stamp":"2026-02-18T07:11:10.766943+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:24:31.181012+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00063668799999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"388'74","reported_seq":883,"reported_epoch":388,"state":"active+clean","last_fresh":"2026-02-18T07:11:12.706795+0000","last_change":"2026-02-18T07:11:12.706795+0000","last_active":"2026-02-18T07:11:12.706795+0000","last_peered":"2026-02-18T07:11:12.706795+0000","last_clean":"2026-02-18T07:11:12.706795+0000","last_became_active":"2026-02-18T06:35:27.374541+0000","last_became_peered":"2026-02-18T06:35:27.374541+0000","last_unstale":"2026-02-18T07:11:12.706795+0000","last_undegraded":"2026-02-18T07:11:12.706795+0000","last_fullsized":"2026-02-18T07:11:12.706795+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"388'74","last_scrub_stamp":"2026-02-18T07:11:12.706756+0000","last_deep_scrub":"388'74","last_deep_scrub_stamp":"2026-02-18T07:11:12.706756+0000","last_clean_scrub_stamp":"2026-02-18T07:11:12.706756+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:45:13.959612+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12910,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738803,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":624000,"kb_used_data":3120,"kb_used_omap":1082,"kb_used_meta":619781,"kb_avail":93747840,"statfs":{"total":96636764160,"available":95997788160,"internally_reserved":0,"allocated":3194880,"data_stored":1711389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1108028,"internal_metadata":634656708},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738804,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":949756,"kb_used_data":3180,"kb_used_omap":378,"kb_used_meta":946181,"kb_avail":93422084,"statfs":{"total":96636764160,"available":95664214016,"internally_reserved":0,"allocated":3256320,"data_stored":2301776,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":387684,"internal_metadata":968889756},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738805,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1348924,"kb_used_data":3180,"kb_used_omap":1178,"kb_used_meta":1344549,"kb_avail":93022916,"statfs":{"total":96636764160,"available":95255465984,"internally_reserved":0,"allocated":3256320,"data_stored":2301776,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1207292,"internal_metadata":1376818180},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6455,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:31.387 DEBUG:teuthology.orchestra.run.trial051:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-18T07:11:31.580 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-18T07:11:31.580 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-18T07:11:31.580 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-18T07:11:31.580 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:31.646 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-18T07:11:31.646 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-18T07:11:31.647 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:31.710 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-18T07:11:31.710 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-18T07:11:31.710 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:31.775 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-18T07:11:31.775 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-18T07:11:31.775 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-18T07:11:31.775 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:31.819 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-18T07:11:31.819 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-18T07:11:31.819 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-18T07:11:31.819 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:31.830 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-18T07:11:31.830 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-18T07:11:31.830 DEBUG:teuthology.orchestra.run.trial051:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-18T07:11:31.899 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T07:11:31.899 DEBUG:teuthology.orchestra.run.trial051:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-18T07:11:32.132 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T07:11:32.132 DEBUG:teuthology.orchestra.run.trial051:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-18T07:11:32.315 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial051.front.sepia.ceph.com 2026-02-18T07:11:32.316 DEBUG:teuthology.orchestra.run.trial051:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-18T07:11:32.487 INFO:tasks.ceph:Archiving mon data... 2026-02-18T07:11:32.488 DEBUG:teuthology.misc:Transferring archived files from trial051:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/data/cluster2.mon.a.tgz 2026-02-18T07:11:32.489 DEBUG:teuthology.orchestra.run.trial051:> mktemp 2026-02-18T07:11:32.504 INFO:teuthology.orchestra.run.trial051.stdout:/tmp/tmp.YQVLGCe63Z 2026-02-18T07:11:32.505 DEBUG:teuthology.orchestra.run.trial051:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.YQVLGCe63Z 2026-02-18T07:11:32.614 DEBUG:teuthology.orchestra.run.trial051:> sudo chmod 0666 /tmp/tmp.YQVLGCe63Z 2026-02-18T07:11:32.693 DEBUG:teuthology.orchestra.remote:trial051:/tmp/tmp.YQVLGCe63Z is 373KB 2026-02-18T07:11:32.754 DEBUG:teuthology.orchestra.run.trial051:> rm -fr /tmp/tmp.YQVLGCe63Z 2026-02-18T07:11:32.770 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-18T07:11:32.770 DEBUG:teuthology.orchestra.run.trial006:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T07:11:32.773 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T07:11:32.834 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-18T07:11:32.846 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-18T07:11:32.846 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T07:11:33.025 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:33.026 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T07:11:33.035 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":1564,"stamp":"2026-02-18T07:11:32.596951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":236760,"kb_used_data":11496,"kb_used_omap":7395,"kb_used_meta":217820,"kb_avail":282878760,"statfs":{"total":289910292480,"available":289667850240,"internally_reserved":0,"allocated":11771904,"data_stored":7427356,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":7573290,"internal_metadata":223047894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000962"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025436+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T07:11:05.025436+0000","last_peered":"2026-02-18T07:11:05.025436+0000","last_clean":"2026-02-18T07:11:05.025436+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T07:11:05.025436+0000","last_undegraded":"2026-02-18T07:11:05.025436+0000","last_fullsized":"2026-02-18T07:11:05.025436+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026046+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T07:11:05.026046+0000","last_peered":"2026-02-18T07:11:05.026046+0000","last_clean":"2026-02-18T07:11:05.026046+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T07:11:05.026046+0000","last_undegraded":"2026-02-18T07:11:05.026046+0000","last_fullsized":"2026-02-18T07:11:05.026046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026072+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T07:11:05.026072+0000","last_peered":"2026-02-18T07:11:05.026072+0000","last_clean":"2026-02-18T07:11:05.026072+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T07:11:05.026072+0000","last_undegraded":"2026-02-18T07:11:05.026072+0000","last_fullsized":"2026-02-18T07:11:05.026072+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025713+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T07:11:05.025713+0000","last_peered":"2026-02-18T07:11:05.025713+0000","last_clean":"2026-02-18T07:11:05.025713+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T07:11:05.025713+0000","last_undegraded":"2026-02-18T07:11:05.025713+0000","last_fullsized":"2026-02-18T07:11:05.025713+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.565944+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T07:11:05.565944+0000","last_peered":"2026-02-18T07:11:05.565944+0000","last_clean":"2026-02-18T07:11:05.565944+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T07:11:05.565944+0000","last_undegraded":"2026-02-18T07:11:05.565944+0000","last_fullsized":"2026-02-18T07:11:05.565944+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":951,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240618+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T07:11:06.240618+0000","last_peered":"2026-02-18T07:11:06.240618+0000","last_clean":"2026-02-18T07:11:06.240618+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T07:11:06.240618+0000","last_undegraded":"2026-02-18T07:11:06.240618+0000","last_fullsized":"2026-02-18T07:11:06.240618+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":952,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240819+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T07:11:06.240819+0000","last_peered":"2026-02-18T07:11:06.240819+0000","last_clean":"2026-02-18T07:11:06.240819+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T07:11:06.240819+0000","last_undegraded":"2026-02-18T07:11:06.240819+0000","last_fullsized":"2026-02-18T07:11:06.240819+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025565+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T07:11:05.025565+0000","last_peered":"2026-02-18T07:11:05.025565+0000","last_clean":"2026-02-18T07:11:05.025565+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T07:11:05.025565+0000","last_undegraded":"2026-02-18T07:11:05.025565+0000","last_fullsized":"2026-02-18T07:11:05.025565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1039,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026942+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T07:11:05.026942+0000","last_peered":"2026-02-18T07:11:05.026942+0000","last_clean":"2026-02-18T07:11:05.026942+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T07:11:05.026942+0000","last_undegraded":"2026-02-18T07:11:05.026942+0000","last_fullsized":"2026-02-18T07:11:05.026942+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66848,"kb_used_data":3792,"kb_used_omap":2121,"kb_used_meta":60918,"kb_avail":94304992,"statfs":{"total":96636764160,"available":96568311808,"internally_reserved":0,"allocated":3883008,"data_stored":2082194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2172534,"internal_metadata":62380426},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3852,"kb_used_omap":2199,"kb_used_meta":60840,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":2252501,"internal_metadata":62300459},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103004,"kb_used_data":3852,"kb_used_omap":3074,"kb_used_meta":96061,"kb_avail":94268836,"statfs":{"total":96636764160,"available":96531288064,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":3148255,"internal_metadata":98367009},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:33.036 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T07:11:33.214 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:33.215 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T07:11:33.225 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":1564,"stamp":"2026-02-18T07:11:32.596951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":236760,"kb_used_data":11496,"kb_used_omap":7395,"kb_used_meta":217820,"kb_avail":282878760,"statfs":{"total":289910292480,"available":289667850240,"internally_reserved":0,"allocated":11771904,"data_stored":7427356,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":7573290,"internal_metadata":223047894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000962"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025436+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T07:11:05.025436+0000","last_peered":"2026-02-18T07:11:05.025436+0000","last_clean":"2026-02-18T07:11:05.025436+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T07:11:05.025436+0000","last_undegraded":"2026-02-18T07:11:05.025436+0000","last_fullsized":"2026-02-18T07:11:05.025436+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026046+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T07:11:05.026046+0000","last_peered":"2026-02-18T07:11:05.026046+0000","last_clean":"2026-02-18T07:11:05.026046+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T07:11:05.026046+0000","last_undegraded":"2026-02-18T07:11:05.026046+0000","last_fullsized":"2026-02-18T07:11:05.026046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026072+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T07:11:05.026072+0000","last_peered":"2026-02-18T07:11:05.026072+0000","last_clean":"2026-02-18T07:11:05.026072+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T07:11:05.026072+0000","last_undegraded":"2026-02-18T07:11:05.026072+0000","last_fullsized":"2026-02-18T07:11:05.026072+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025713+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T07:11:05.025713+0000","last_peered":"2026-02-18T07:11:05.025713+0000","last_clean":"2026-02-18T07:11:05.025713+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T07:11:05.025713+0000","last_undegraded":"2026-02-18T07:11:05.025713+0000","last_fullsized":"2026-02-18T07:11:05.025713+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.565944+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T07:11:05.565944+0000","last_peered":"2026-02-18T07:11:05.565944+0000","last_clean":"2026-02-18T07:11:05.565944+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T07:11:05.565944+0000","last_undegraded":"2026-02-18T07:11:05.565944+0000","last_fullsized":"2026-02-18T07:11:05.565944+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":951,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240618+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T07:11:06.240618+0000","last_peered":"2026-02-18T07:11:06.240618+0000","last_clean":"2026-02-18T07:11:06.240618+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T07:11:06.240618+0000","last_undegraded":"2026-02-18T07:11:06.240618+0000","last_fullsized":"2026-02-18T07:11:06.240618+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":952,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240819+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T07:11:06.240819+0000","last_peered":"2026-02-18T07:11:06.240819+0000","last_clean":"2026-02-18T07:11:06.240819+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T07:11:06.240819+0000","last_undegraded":"2026-02-18T07:11:06.240819+0000","last_fullsized":"2026-02-18T07:11:06.240819+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025565+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T07:11:05.025565+0000","last_peered":"2026-02-18T07:11:05.025565+0000","last_clean":"2026-02-18T07:11:05.025565+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T07:11:05.025565+0000","last_undegraded":"2026-02-18T07:11:05.025565+0000","last_fullsized":"2026-02-18T07:11:05.025565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1039,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026942+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T07:11:05.026942+0000","last_peered":"2026-02-18T07:11:05.026942+0000","last_clean":"2026-02-18T07:11:05.026942+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T07:11:05.026942+0000","last_undegraded":"2026-02-18T07:11:05.026942+0000","last_fullsized":"2026-02-18T07:11:05.026942+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66848,"kb_used_data":3792,"kb_used_omap":2121,"kb_used_meta":60918,"kb_avail":94304992,"statfs":{"total":96636764160,"available":96568311808,"internally_reserved":0,"allocated":3883008,"data_stored":2082194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2172534,"internal_metadata":62380426},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3852,"kb_used_omap":2199,"kb_used_meta":60840,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":2252501,"internal_metadata":62300459},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103004,"kb_used_data":3852,"kb_used_omap":3074,"kb_used_meta":96061,"kb_avail":94268836,"statfs":{"total":96636764160,"available":96531288064,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":3148255,"internal_metadata":98367009},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:33.226 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-18T07:11:33.226 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T07:11:33.408 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:33.409 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T07:11:33.417 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":1564,"stamp":"2026-02-18T07:11:32.596951+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":236760,"kb_used_data":11496,"kb_used_omap":7395,"kb_used_meta":217820,"kb_avail":282878760,"statfs":{"total":289910292480,"available":289667850240,"internally_reserved":0,"allocated":11771904,"data_stored":7427356,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":7573290,"internal_metadata":223047894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000962"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025436+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T07:11:05.025436+0000","last_peered":"2026-02-18T07:11:05.025436+0000","last_clean":"2026-02-18T07:11:05.025436+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T07:11:05.025436+0000","last_undegraded":"2026-02-18T07:11:05.025436+0000","last_fullsized":"2026-02-18T07:11:05.025436+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026046+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T07:11:05.026046+0000","last_peered":"2026-02-18T07:11:05.026046+0000","last_clean":"2026-02-18T07:11:05.026046+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T07:11:05.026046+0000","last_undegraded":"2026-02-18T07:11:05.026046+0000","last_fullsized":"2026-02-18T07:11:05.026046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026072+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T07:11:05.026072+0000","last_peered":"2026-02-18T07:11:05.026072+0000","last_clean":"2026-02-18T07:11:05.026072+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T07:11:05.026072+0000","last_undegraded":"2026-02-18T07:11:05.026072+0000","last_fullsized":"2026-02-18T07:11:05.026072+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025713+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T07:11:05.025713+0000","last_peered":"2026-02-18T07:11:05.025713+0000","last_clean":"2026-02-18T07:11:05.025713+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T07:11:05.025713+0000","last_undegraded":"2026-02-18T07:11:05.025713+0000","last_fullsized":"2026-02-18T07:11:05.025713+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.565944+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T07:11:05.565944+0000","last_peered":"2026-02-18T07:11:05.565944+0000","last_clean":"2026-02-18T07:11:05.565944+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T07:11:05.565944+0000","last_undegraded":"2026-02-18T07:11:05.565944+0000","last_fullsized":"2026-02-18T07:11:05.565944+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":951,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240618+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T07:11:06.240618+0000","last_peered":"2026-02-18T07:11:06.240618+0000","last_clean":"2026-02-18T07:11:06.240618+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T07:11:06.240618+0000","last_undegraded":"2026-02-18T07:11:06.240618+0000","last_fullsized":"2026-02-18T07:11:06.240618+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":952,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240819+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T07:11:06.240819+0000","last_peered":"2026-02-18T07:11:06.240819+0000","last_clean":"2026-02-18T07:11:06.240819+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T07:11:06.240819+0000","last_undegraded":"2026-02-18T07:11:06.240819+0000","last_fullsized":"2026-02-18T07:11:06.240819+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025565+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T07:11:05.025565+0000","last_peered":"2026-02-18T07:11:05.025565+0000","last_clean":"2026-02-18T07:11:05.025565+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T07:11:05.025565+0000","last_undegraded":"2026-02-18T07:11:05.025565+0000","last_fullsized":"2026-02-18T07:11:05.025565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1039,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026942+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T07:11:05.026942+0000","last_peered":"2026-02-18T07:11:05.026942+0000","last_clean":"2026-02-18T07:11:05.026942+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T07:11:05.026942+0000","last_undegraded":"2026-02-18T07:11:05.026942+0000","last_fullsized":"2026-02-18T07:11:05.026942+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66848,"kb_used_data":3792,"kb_used_omap":2121,"kb_used_meta":60918,"kb_avail":94304992,"statfs":{"total":96636764160,"available":96568311808,"internally_reserved":0,"allocated":3883008,"data_stored":2082194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2172534,"internal_metadata":62380426},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3852,"kb_used_omap":2199,"kb_used_meta":60840,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":2252501,"internal_metadata":62300459},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103004,"kb_used_data":3852,"kb_used_omap":3074,"kb_used_meta":96061,"kb_avail":94268836,"statfs":{"total":96636764160,"available":96531288064,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":3148255,"internal_metadata":98367009},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:33.419 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-18T07:11:33.599 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:33.599 INFO:teuthology.orchestra.run.trial006.stdout:{"epoch":472,"fsid":"8af030b1-7761-4bce-8b14-c3dac9d5a86a","created":"2026-02-18T06:34:57.835085+0000","modified":"2026-02-18T07:11:05.019382+0000","last_up_change":"2026-02-18T06:35:01.841544+0000","last_in_change":"2026-02-18T06:34:58.508788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-18T06:35:02.294274+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-18T06:35:05.901344+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4406057c-7381-449c-a6a8-0e45373e3429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6809","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6811","nonce":660097312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6813","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6815","nonce":660097312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6821","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6823","nonce":660097312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6817","nonce":660097312},{"type":"v1","addr":"10.20.193.6:6819","nonce":660097312}]},"public_addr":"10.20.193.6:6811/660097312","cluster_addr":"10.20.193.6:6815/660097312","heartbeat_back_addr":"10.20.193.6:6823/660097312","heartbeat_front_addr":"10.20.193.6:6819/660097312","state":["exists","up"]},{"osd":1,"uuid":"f61faa8c-808b-4525-8a50-f48f1ff0ed96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6808","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6810","nonce":1636389907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6812","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6814","nonce":1636389907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6820","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6822","nonce":1636389907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6816","nonce":1636389907},{"type":"v1","addr":"10.20.193.6:6818","nonce":1636389907}]},"public_addr":"10.20.193.6:6810/1636389907","cluster_addr":"10.20.193.6:6814/1636389907","heartbeat_back_addr":"10.20.193.6:6822/1636389907","heartbeat_front_addr":"10.20.193.6:6818/1636389907","state":["exists","up"]},{"osd":2,"uuid":"53a83bfa-9f75-4cd4-937c-4464633a9dc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6800","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6801","nonce":1693096825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6802","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6803","nonce":1693096825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6806","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6807","nonce":1693096825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.6:6804","nonce":1693096825},{"type":"v1","addr":"10.20.193.6:6805","nonce":1693096825}]},"public_addr":"10.20.193.6:6801/1693096825","cluster_addr":"10.20.193.6:6803/1693096825","heartbeat_back_addr":"10.20.193.6:6807/1693096825","heartbeat_front_addr":"10.20.193.6:6805/1693096825","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:00.434953+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:00.439267+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-18T06:35:00.435135+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-18T07:11:34.609 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-18T07:11:34.610 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:34.696 INFO:teuthology.orchestra.run.trial006.stdout:{ 2026-02-18T07:11:34.697 INFO:teuthology.orchestra.run.trial006.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:34.697 INFO:teuthology.orchestra.run.trial006.stdout:} 2026-02-18T07:11:34.703 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-18T07:11:34.885 INFO:teuthology.orchestra.run.trial006.stderr:instructed osd(s) 0 to deep-scrub 2026-02-18T07:11:34.894 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-18T07:11:34.895 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:34.977 INFO:teuthology.orchestra.run.trial006.stdout:{ 2026-02-18T07:11:34.977 INFO:teuthology.orchestra.run.trial006.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:34.978 INFO:teuthology.orchestra.run.trial006.stdout:} 2026-02-18T07:11:34.983 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-18T07:11:35.160 INFO:teuthology.orchestra.run.trial006.stderr:instructed osd(s) 1 to deep-scrub 2026-02-18T07:11:35.169 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-18T07:11:35.169 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-18T07:11:35.251 INFO:teuthology.orchestra.run.trial006.stdout:{ 2026-02-18T07:11:35.252 INFO:teuthology.orchestra.run.trial006.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-18T07:11:35.252 INFO:teuthology.orchestra.run.trial006.stdout:} 2026-02-18T07:11:35.259 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-18T07:11:35.443 INFO:teuthology.orchestra.run.trial006.stderr:instructed osd(s) 2 to deep-scrub 2026-02-18T07:11:35.453 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T07:11:35.632 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:35.633 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T07:11:35.640 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":1565,"stamp":"2026-02-18T07:11:34.597226+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":236760,"kb_used_data":11496,"kb_used_omap":7395,"kb_used_meta":217820,"kb_avail":282878760,"statfs":{"total":289910292480,"available":289667850240,"internally_reserved":0,"allocated":11771904,"data_stored":7427356,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":7573290,"internal_metadata":223047894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001095"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025436+0000","last_change":"2026-02-18T06:35:09.302013+0000","last_active":"2026-02-18T07:11:05.025436+0000","last_peered":"2026-02-18T07:11:05.025436+0000","last_clean":"2026-02-18T07:11:05.025436+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T07:11:05.025436+0000","last_undegraded":"2026-02-18T07:11:05.025436+0000","last_fullsized":"2026-02-18T07:11:05.025436+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:38:03.266969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":946,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026046+0000","last_change":"2026-02-18T06:35:09.301343+0000","last_active":"2026-02-18T07:11:05.026046+0000","last_peered":"2026-02-18T07:11:05.026046+0000","last_clean":"2026-02-18T07:11:05.026046+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T07:11:05.026046+0000","last_undegraded":"2026-02-18T07:11:05.026046+0000","last_fullsized":"2026-02-18T07:11:05.026046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:38:25.134357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026072+0000","last_change":"2026-02-18T06:35:09.302004+0000","last_active":"2026-02-18T07:11:05.026072+0000","last_peered":"2026-02-18T07:11:05.026072+0000","last_clean":"2026-02-18T07:11:05.026072+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T07:11:05.026072+0000","last_undegraded":"2026-02-18T07:11:05.026072+0000","last_fullsized":"2026-02-18T07:11:05.026072+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T10:37:17.282403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":947,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025713+0000","last_change":"2026-02-18T06:35:09.301714+0000","last_active":"2026-02-18T07:11:05.025713+0000","last_peered":"2026-02-18T07:11:05.025713+0000","last_clean":"2026-02-18T07:11:05.025713+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T07:11:05.025713+0000","last_undegraded":"2026-02-18T07:11:05.025713+0000","last_fullsized":"2026-02-18T07:11:05.025713+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:45:29.958104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.565944+0000","last_change":"2026-02-18T06:35:09.300117+0000","last_active":"2026-02-18T07:11:05.565944+0000","last_peered":"2026-02-18T07:11:05.565944+0000","last_clean":"2026-02-18T07:11:05.565944+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T07:11:05.565944+0000","last_undegraded":"2026-02-18T07:11:05.565944+0000","last_fullsized":"2026-02-18T07:11:05.565944+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T13:27:05.891417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":951,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240618+0000","last_change":"2026-02-18T06:35:09.298247+0000","last_active":"2026-02-18T07:11:06.240618+0000","last_peered":"2026-02-18T07:11:06.240618+0000","last_clean":"2026-02-18T07:11:06.240618+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T07:11:06.240618+0000","last_undegraded":"2026-02-18T07:11:06.240618+0000","last_fullsized":"2026-02-18T07:11:06.240618+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T07:00:21.001620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":952,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:06.240819+0000","last_change":"2026-02-18T06:35:09.298246+0000","last_active":"2026-02-18T07:11:06.240819+0000","last_peered":"2026-02-18T07:11:06.240819+0000","last_clean":"2026-02-18T07:11:06.240819+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T07:11:06.240819+0000","last_undegraded":"2026-02-18T07:11:06.240819+0000","last_fullsized":"2026-02-18T07:11:06.240819+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:54:46.179502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":948,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.025565+0000","last_change":"2026-02-18T06:35:09.301658+0000","last_active":"2026-02-18T07:11:05.025565+0000","last_peered":"2026-02-18T07:11:05.025565+0000","last_clean":"2026-02-18T07:11:05.025565+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T07:11:05.025565+0000","last_undegraded":"2026-02-18T07:11:05.025565+0000","last_fullsized":"2026-02-18T07:11:05.025565+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:06.288451+0000","last_clean_scrub_stamp":"2026-02-18T06:35:06.288451+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:32:09.716046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1039,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:05.026942+0000","last_change":"2026-02-18T06:35:04.288750+0000","last_active":"2026-02-18T07:11:05.026942+0000","last_peered":"2026-02-18T07:11:05.026942+0000","last_clean":"2026-02-18T07:11:05.026942+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T07:11:05.026942+0000","last_undegraded":"2026-02-18T07:11:05.026942+0000","last_fullsized":"2026-02-18T07:11:05.026942+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T06:35:03.285309+0000","last_clean_scrub_stamp":"2026-02-18T06:35:03.285309+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:26:22.062774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738808,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66848,"kb_used_data":3792,"kb_used_omap":2121,"kb_used_meta":60918,"kb_avail":94304992,"statfs":{"total":96636764160,"available":96568311808,"internally_reserved":0,"allocated":3883008,"data_stored":2082194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2172534,"internal_metadata":62380426},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3852,"kb_used_omap":2199,"kb_used_meta":60840,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":2252501,"internal_metadata":62300459},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738811,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103004,"kb_used_data":3852,"kb_used_omap":3074,"kb_used_meta":96061,"kb_avail":94268836,"statfs":{"total":96636764160,"available":96531288064,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":3148255,"internal_metadata":98367009},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:35.642 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.643 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.643 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.643 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.643 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.643 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.644 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.644 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-18T06:35:06.288451+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.644 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-18T06:35:03.285309+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=6, tm_min=35, tm_sec=3, tm_wday=2, tm_yday=49, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=18, tm_hour=7, tm_min=11, tm_sec=33, tm_wday=2, tm_yday=49, tm_isdst=0) 2026-02-18T07:11:35.644 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-18T07:11:55.644 DEBUG:teuthology.orchestra.run.trial006:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-18T07:11:55.824 INFO:teuthology.orchestra.run.trial006.stdout: 2026-02-18T07:11:55.824 INFO:teuthology.orchestra.run.trial006.stderr:dumped all 2026-02-18T07:11:55.833 INFO:teuthology.orchestra.run.trial006.stdout:{"pg_ready":true,"pg_map":{"version":1575,"stamp":"2026-02-18T07:11:54.599241+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":236760,"kb_used_data":11496,"kb_used_omap":7395,"kb_used_meta":217820,"kb_avail":282878760,"statfs":{"total":289910292480,"available":289667850240,"internally_reserved":0,"allocated":11771904,"data_stored":7427356,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":7573290,"internal_metadata":223047894},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000965"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":954,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:41.228789+0000","last_change":"2026-02-18T07:11:41.228789+0000","last_active":"2026-02-18T07:11:41.228789+0000","last_peered":"2026-02-18T07:11:41.228789+0000","last_clean":"2026-02-18T07:11:41.228789+0000","last_became_active":"2026-02-18T06:35:07.297738+0000","last_became_peered":"2026-02-18T06:35:07.297738+0000","last_unstale":"2026-02-18T07:11:41.228789+0000","last_undegraded":"2026-02-18T07:11:41.228789+0000","last_fullsized":"2026-02-18T07:11:41.228789+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:41.228747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:41.228747+0000","last_clean_scrub_stamp":"2026-02-18T07:11:41.228747+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:52:05.847978+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.001028272,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":954,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:37.144361+0000","last_change":"2026-02-18T07:11:37.144361+0000","last_active":"2026-02-18T07:11:37.144361+0000","last_peered":"2026-02-18T07:11:37.144361+0000","last_clean":"2026-02-18T07:11:37.144361+0000","last_became_active":"2026-02-18T06:35:07.297916+0000","last_became_peered":"2026-02-18T06:35:07.297916+0000","last_unstale":"2026-02-18T07:11:37.144361+0000","last_undegraded":"2026-02-18T07:11:37.144361+0000","last_fullsized":"2026-02-18T07:11:37.144361+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:37.143819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:37.143819+0000","last_clean_scrub_stamp":"2026-02-18T07:11:37.143819+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T18:47:01.476003+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00048496699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":955,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:39.196460+0000","last_change":"2026-02-18T07:11:39.196460+0000","last_active":"2026-02-18T07:11:39.196460+0000","last_peered":"2026-02-18T07:11:39.196460+0000","last_clean":"2026-02-18T07:11:39.196460+0000","last_became_active":"2026-02-18T06:35:07.297798+0000","last_became_peered":"2026-02-18T06:35:07.297798+0000","last_unstale":"2026-02-18T07:11:39.196460+0000","last_undegraded":"2026-02-18T07:11:39.196460+0000","last_fullsized":"2026-02-18T07:11:39.196460+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:39.196408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:39.196408+0000","last_clean_scrub_stamp":"2026-02-18T07:11:39.196408+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:19:54.050528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001118469,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":955,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:38.148921+0000","last_change":"2026-02-18T07:11:38.148921+0000","last_active":"2026-02-18T07:11:38.148921+0000","last_peered":"2026-02-18T07:11:38.148921+0000","last_clean":"2026-02-18T07:11:38.148921+0000","last_became_active":"2026-02-18T06:35:07.297861+0000","last_became_peered":"2026-02-18T06:35:07.297861+0000","last_unstale":"2026-02-18T07:11:38.148921+0000","last_undegraded":"2026-02-18T07:11:38.148921+0000","last_fullsized":"2026-02-18T07:11:38.148921+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:38.148878+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:38.148878+0000","last_clean_scrub_stamp":"2026-02-18T07:11:38.148878+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:55:38.200791+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00083667799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"472'3","reported_seq":957,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:35.885717+0000","last_change":"2026-02-18T07:11:35.885717+0000","last_active":"2026-02-18T07:11:35.885717+0000","last_peered":"2026-02-18T07:11:35.885717+0000","last_clean":"2026-02-18T07:11:35.885717+0000","last_became_active":"2026-02-18T06:35:07.297727+0000","last_became_peered":"2026-02-18T06:35:07.297727+0000","last_unstale":"2026-02-18T07:11:35.885717+0000","last_undegraded":"2026-02-18T07:11:35.885717+0000","last_fullsized":"2026-02-18T07:11:35.885717+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"472'3","last_scrub_stamp":"2026-02-18T07:11:35.885622+0000","last_deep_scrub":"472'3","last_deep_scrub_stamp":"2026-02-18T07:11:35.885622+0000","last_clean_scrub_stamp":"2026-02-18T07:11:35.885622+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T11:16:00.302105+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.000619105,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":959,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:36.788873+0000","last_change":"2026-02-18T07:11:36.788873+0000","last_active":"2026-02-18T07:11:36.788873+0000","last_peered":"2026-02-18T07:11:36.788873+0000","last_clean":"2026-02-18T07:11:36.788873+0000","last_became_active":"2026-02-18T06:35:07.297585+0000","last_became_peered":"2026-02-18T06:35:07.297585+0000","last_unstale":"2026-02-18T07:11:36.788873+0000","last_undegraded":"2026-02-18T07:11:36.788873+0000","last_fullsized":"2026-02-18T07:11:36.788873+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:36.788824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:36.788824+0000","last_clean_scrub_stamp":"2026-02-18T07:11:36.788824+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T16:48:46.663930+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00042561699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":960,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:35.792624+0000","last_change":"2026-02-18T07:11:35.792624+0000","last_active":"2026-02-18T07:11:35.792624+0000","last_peered":"2026-02-18T07:11:35.792624+0000","last_clean":"2026-02-18T07:11:35.792624+0000","last_became_active":"2026-02-18T06:35:07.297670+0000","last_became_peered":"2026-02-18T06:35:07.297670+0000","last_unstale":"2026-02-18T07:11:35.792624+0000","last_undegraded":"2026-02-18T07:11:35.792624+0000","last_fullsized":"2026-02-18T07:11:35.792624+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-18T07:11:35.792570+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-18T07:11:35.792570+0000","last_clean_scrub_stamp":"2026-02-18T07:11:35.792570+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T09:38:39.990638+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00033093199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"472'2","reported_seq":957,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:36.146636+0000","last_change":"2026-02-18T07:11:36.146636+0000","last_active":"2026-02-18T07:11:36.146636+0000","last_peered":"2026-02-18T07:11:36.146636+0000","last_clean":"2026-02-18T07:11:36.146636+0000","last_became_active":"2026-02-18T06:35:07.297234+0000","last_became_peered":"2026-02-18T06:35:07.297234+0000","last_unstale":"2026-02-18T07:11:36.146636+0000","last_undegraded":"2026-02-18T07:11:36.146636+0000","last_fullsized":"2026-02-18T07:11:36.146636+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"472'2","last_scrub_stamp":"2026-02-18T07:11:36.146607+0000","last_deep_scrub":"472'2","last_deep_scrub_stamp":"2026-02-18T07:11:36.146607+0000","last_clean_scrub_stamp":"2026-02-18T07:11:36.146607+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T17:23:15.476231+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00060616100000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"472'74","reported_seq":1049,"reported_epoch":472,"state":"active+clean","last_fresh":"2026-02-18T07:11:40.185934+0000","last_change":"2026-02-18T07:11:40.185934+0000","last_active":"2026-02-18T07:11:40.185934+0000","last_peered":"2026-02-18T07:11:40.185934+0000","last_clean":"2026-02-18T07:11:40.185934+0000","last_became_active":"2026-02-18T06:35:04.288166+0000","last_became_peered":"2026-02-18T06:35:04.288166+0000","last_unstale":"2026-02-18T07:11:40.185934+0000","last_undegraded":"2026-02-18T07:11:40.185934+0000","last_fullsized":"2026-02-18T07:11:40.185934+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"472'74","last_scrub_stamp":"2026-02-18T07:11:40.185899+0000","last_deep_scrub":"472'74","last_deep_scrub_stamp":"2026-02-18T07:11:40.185899+0000","last_clean_scrub_stamp":"2026-02-18T07:11:40.185899+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-19T08:14:44.894139+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":12906,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738812,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66848,"kb_used_data":3792,"kb_used_omap":2121,"kb_used_meta":60918,"kb_avail":94304992,"statfs":{"total":96636764160,"available":96568311808,"internally_reserved":0,"allocated":3883008,"data_stored":2082194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2172534,"internal_metadata":62380426},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738813,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3852,"kb_used_omap":2199,"kb_used_meta":60840,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":2252501,"internal_metadata":62300459},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738815,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103004,"kb_used_data":3852,"kb_used_omap":3074,"kb_used_meta":96061,"kb_avail":94268836,"statfs":{"total":96636764160,"available":96531288064,"internally_reserved":0,"allocated":3944448,"data_stored":2672581,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":3148255,"internal_metadata":98367009},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":6453,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-18T07:11:55.834 DEBUG:teuthology.orchestra.run.trial006:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-18T07:11:56.026 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-18T07:11:56.026 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-18T07:11:56.026 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-18T07:11:56.026 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:56.076 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-18T07:11:56.076 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-18T07:11:56.076 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:56.116 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-18T07:11:56.116 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-18T07:11:56.117 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:56.188 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-18T07:11:56.189 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-18T07:11:56.189 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-18T07:11:56.189 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:56.260 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-18T07:11:56.261 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-18T07:11:56.261 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-18T07:11:56.261 INFO:teuthology.orchestra.run:waiting for 300 2026-02-18T07:11:56.313 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.mon.a is failed for ~0s 2026-02-18T07:11:56.313 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-18T07:11:56.314 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-18T07:11:56.314 DEBUG:teuthology.orchestra.run.trial006:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-18T07:11:56.379 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T07:11:56.379 DEBUG:teuthology.orchestra.run.trial006:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-18T07:11:56.502 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T07:11:56.502 DEBUG:teuthology.orchestra.run.trial006:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-18T07:11:56.541 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial006.front.sepia.ceph.com 2026-02-18T07:11:56.541 DEBUG:teuthology.orchestra.run.trial006:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-18T07:11:56.618 INFO:tasks.ceph:Archiving mon data... 2026-02-18T07:11:56.618 DEBUG:teuthology.misc:Transferring archived files from trial006:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/data/cluster1.mon.a.tgz 2026-02-18T07:11:56.618 DEBUG:teuthology.orchestra.run.trial006:> mktemp 2026-02-18T07:11:56.674 INFO:teuthology.orchestra.run.trial006.stdout:/tmp/tmp.c9CycukmRj 2026-02-18T07:11:56.675 DEBUG:teuthology.orchestra.run.trial006:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.c9CycukmRj 2026-02-18T07:11:56.791 DEBUG:teuthology.orchestra.run.trial006:> sudo chmod 0666 /tmp/tmp.c9CycukmRj 2026-02-18T07:11:56.878 DEBUG:teuthology.orchestra.remote:trial006:/tmp/tmp.c9CycukmRj is 414KB 2026-02-18T07:11:56.955 DEBUG:teuthology.orchestra.run.trial006:> rm -fr /tmp/tmp.c9CycukmRj 2026-02-18T07:11:56.972 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-18T07:11:56.972 DEBUG:teuthology.orchestra.run.trial006:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T07:11:57.013 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-18T07:11:57.093 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-18T07:11:57.109 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-18T07:11:57.109 INFO:tasks.ceph:Archiving crash dumps... 2026-02-18T07:11:57.112 DEBUG:teuthology.misc:Transferring archived files from trial006:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial006/crash 2026-02-18T07:11:57.113 DEBUG:teuthology.orchestra.run.trial006:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-18T07:11:57.158 DEBUG:teuthology.misc:Transferring archived files from trial051:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial051/crash 2026-02-18T07:11:57.160 DEBUG:teuthology.orchestra.run.trial051:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-18T07:11:57.184 INFO:tasks.ceph:Compressing logs... 2026-02-18T07:11:57.185 DEBUG:teuthology.orchestra.run.trial006:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T07:11:57.205 DEBUG:teuthology.orchestra.run.trial051:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T07:11:57.226 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30386.log 2026-02-18T07:11:57.242 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25031.log 2026-02-18T07:11:57.242 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30484.log 2026-02-18T07:11:57.242 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30386.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30386.log.gz 2026-02-18T07:11:57.242 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.25031.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25031.log.gz 2026-02-18T07:11:57.242 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24904.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30484.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28411.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30484.log.gz 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30533.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.24904.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.24904.log.gz 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30141.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28411.log.gz 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30533.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30533.log.gz 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28120.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30141.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27921.log 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30141.log.gz 2026-02-18T07:11:57.243 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28120.log.gz 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27921.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27921.log.gz 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28019.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28019.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27782.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28019.log.gz 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24810.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27782.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27782.log.gz 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24857.log 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.24810.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.24810.log.gz 2026-02-18T07:11:57.244 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.24857.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.24857.log.gz 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30288.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30582.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30288.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30288.log.gz 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24581.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24676.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30582.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30582.log.gz 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.24581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28268.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.24581.log.gz 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.24676.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.24676.log.gz 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30679.log 2026-02-18T07:11:57.245 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27807.log 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28268.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28268.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30679.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30679.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30632.log 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.27807.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28362.log 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30632.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30632.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28460.log 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30435.log 0.0% 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.28362.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28460.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28460.log.gz 2026-02-18T07:11:57.246 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28269.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30435.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.19723.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30435.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28269.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28269.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/ceph.tmp-client.admin.19723.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.19723.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30337.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28122.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30337.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30337.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27970.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28122.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28121.log 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28122.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27970.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27970.log.gz 2026-02-18T07:11:57.247 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25148.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30190.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28121.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.25148.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.25148.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27872.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr: 93.0% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30190.log: gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30190.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27872.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27872.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27712.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30239.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.27712.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.27712.log.gz 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28267.log 2026-02-18T07:11:57.248 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30239.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30239.log.gz 2026-02-18T07:11:57.249 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30092.log 2026-02-18T07:11:57.249 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.28267.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.28267.log.gz 2026-02-18T07:11:57.249 INFO:teuthology.orchestra.run.trial006.stderr:/var/log/ceph/cluster1-client.admin.30092.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30092.log.gz 2026-02-18T07:11:57.249 INFO:teuthology.orchestra.run.trial006.stderr: 87.7% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-18T07:11:57.252 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60586.log 2026-02-18T07:11:57.253 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.66181.log 2026-02-18T07:11:57.284 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55881.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60586.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.66181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41248.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66290.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65505.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41248.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50300.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55881.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65505.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35618.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65505.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42604.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66290.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.66181.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50300.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50895.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35618.log.gz 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33149.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32995.log 2026-02-18T07:11:57.285 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42604.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.33149.log: -- replaced with /var/log/ceph/cluster1-client.mirror.50895.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40439.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67437.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33149.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.32995.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65000.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40439.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54730.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67437.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65000.log: 78.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65000.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46887.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58434.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54730.log: /var/log/ceph/cluster2-client.mirror.46887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39765.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54730.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37978.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.39765.log: --verbose -- /var/log/ceph/cluster2-client.mirror.34366.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58434.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46887.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39765.log.gz 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34726.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63485.log 2026-02-18T07:11:57.286 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50276.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34366.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57482.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63485.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50276.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57462.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50276.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59477.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57462.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.41612.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57462.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45684.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57482.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.41612.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59477.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34998.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37978.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63029.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34726.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48039.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45684.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34998.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63029.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64233.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63029.log.gz 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48039.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32489.log 2026-02-18T07:11:57.287 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48039.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40656.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.64233.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53471.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32489.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50570.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60518.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53471.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58785.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63051.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60518.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57441.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50570.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58703.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66727.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58785.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58703.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57441.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64645.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58703.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42938.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66727.log.gz 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50871.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42938.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51924.log 2026-02-18T07:11:57.288 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42938.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50871.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31065.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50871.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52556.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51924.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.31065.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33573.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 71.2% -- replaced with /var/log/ceph/cluster2-client.admin.31065.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30638.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43101.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.33573.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51736.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.30638.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52556.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61891.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51736.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.29927.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43101.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40950.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61891.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.0.29927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46746.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40950.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51736.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64360.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40950.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40535.log 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46746.log.gz 2026-02-18T07:11:57.289 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49336.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64360.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.29927.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38831.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42750.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40535.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49336.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53815.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38831.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38236.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63801.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38236.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66913.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53815.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63801.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.41612.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60338.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63801.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46348.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66913.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41312.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60338.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60338.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28880.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46348.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41312.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56221.log 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41312.log.gz 2026-02-18T07:11:57.290 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55403.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28880.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28880.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40476.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56221.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55403.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61216.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38236.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51517.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61216.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40476.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49987.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61216.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43010.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49987.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55403.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48087.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51517.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49987.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33922.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43010.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68157.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48087.log: /var/log/ceph/cluster2-client.mirror.33922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65442.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48087.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68157.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40456.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68157.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33922.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41825.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65442.log.gz 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50457.log 2026-02-18T07:11:57.291 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40456.log: /var/log/ceph/cluster2-client.mirror.41825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40253.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40456.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67019.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50457.log: /var/log/ceph/cluster2-client.mirror.40253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31082.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50457.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58475.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67019.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31082.log: 63.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31082.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41268.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65887.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58475.log: /var/log/ceph/cluster2-client.mirror.41268.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37998.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65887.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41268.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65887.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51716.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58475.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43121.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64401.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40555.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43121.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34117.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64401.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54253.log 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51716.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34117.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40555.log.gz 2026-02-18T07:11:57.292 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43084.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.34117.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.64000.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66827.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54253.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43084.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.2.64000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54595.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37998.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66827.log: 87.5% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46226.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54595.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33127.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54595.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59808.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68206.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46226.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54002.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59808.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65797.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68206.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68500.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56945.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 83.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65797.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68500.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.2.64000.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68500.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29122.log 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54002.log.gz 2026-02-18T07:11:57.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40377.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29122.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56945.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54619.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29122.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42800.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45724.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.33127.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54619.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40377.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42800.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54619.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39621.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42800.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38994.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39621.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.20161.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45724.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38994.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39621.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41371.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/ceph.tmp-client.admin.20161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31150.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.20161.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57317.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65687.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 64.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31150.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57317.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36245.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54347.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65687.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57317.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38465.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48710.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54347.log: /var/log/ceph/cluster2-client.mirror.38465.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34069.log 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54347.log.gz 2026-02-18T07:11:57.294 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48710.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38465.log.gz 2026-02-18T07:11:57.295 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45750.log 2026-02-18T07:11:57.295 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48710.log.gz 2026-02-18T07:11:57.295 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57854.log 2026-02-18T07:11:57.295 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67244.log 2026-02-18T07:11:57.295 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45750.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34069.log.gz 2026-02-18T07:11:57.302 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57854.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45750.log.gz 2026-02-18T07:11:57.302 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65089.log 2026-02-18T07:11:57.302 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65251.log 2026-02-18T07:11:57.302 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57854.log.gz 2026-02-18T07:11:57.302 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46372.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67244.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36245.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65089.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65422.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65251.log: /var/log/ceph/cluster1-client.mirror.46372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54297.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65251.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.46372.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55923.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65422.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43235.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54297.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60631.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55923.log: /var/log/ceph/cluster2-client.mirror.43235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57751.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55923.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46300.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60631.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40194.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57751.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42993.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46300.log: /var/log/ceph/cluster2-client.mirror.40194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65295.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46300.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40194.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36701.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42993.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53558.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64914.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65295.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36701.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67539.log 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53558.log.gz 2026-02-18T07:11:57.303 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31277.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64914.log: /var/log/ceph/cluster2-client.mirror.67539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36383.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64914.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67539.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50529.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31277.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36887.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.36383.log: /var/log/ceph/cluster1-client.mirror.50529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46794.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.36383.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50529.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33499.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57648.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46794.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33499.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59849.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57648.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58185.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33499.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57648.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50771.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59849.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58185.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67039.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50771.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58185.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60473.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67039.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51249.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.67039.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47312.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60473.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60473.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51249.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50771.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40614.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47312.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67519.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.47312.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60721.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67519.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40614.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39083.log 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51249.log.gz 2026-02-18T07:11:57.304 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67519.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.49749.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60721.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39083.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46621.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.49749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41349.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46621.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44674.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46621.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52194.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41349.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44674.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40597.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46182.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.40597.log: --verbose -- /var/log/ceph/cluster2-client.mirror.47108.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40597.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62602.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44674.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47108.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46182.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42708.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52194.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53349.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62602.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30060.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60766.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47108.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42708.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.30060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55901.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 78.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30060.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53349.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66181.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60766.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58723.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45164.log 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr: 81.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66181.log.gz 2026-02-18T07:11:57.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68304.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45164.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35369.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55901.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59394.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58723.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68304.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.35369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44754.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.35369.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45164.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59394.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33869.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43401.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44754.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.33869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35176.log 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59394.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.33869.log.gz 2026-02-18T07:11:57.306 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56483.log 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35176.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43401.log.gz 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56667.log 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35176.log.gz 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64340.log 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64189.log 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56667.log.gz 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65628.log 2026-02-18T07:11:57.308 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64340.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64189.log: gzip 92.4% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.56483.log.gz -- 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: /var/log/ceph/cluster2-client.mirror.37274.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28420.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65628.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41882.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37274.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51184.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.admin.28420.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60563.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41882.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64210.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51184.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60563.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.68059.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60563.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64210.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46662.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64210.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.68059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25570.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 59.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.68059.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38041.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58040.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25570.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51852.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50228.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46662.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44696.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51852.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50228.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63366.log 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58040.log.gz 2026-02-18T07:11:57.309 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50228.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68694.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63366.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44696.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50407.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63366.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38021.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68694.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61373.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38021.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50407.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64298.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39881.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.38041.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.61373.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65931.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64298.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39881.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49288.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39881.log.gz 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65931.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47039.log 2026-02-18T07:11:57.311 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65931.log.gz 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53178.log 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47039.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30672.log 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.49288.log.gz 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47039.log.gz 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32177.log 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53178.log.gz 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30672.log: gzip -5 --verbose -- 63.8% /var/log/ceph/cluster1-client.mirror.61553.log 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.30672.log.gz 2026-02-18T07:11:57.312 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32177.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66848.log 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38021.log.gz 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61553.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58909.log 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32177.log.gz 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61553.log.gz 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66848.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39861.log 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.66848.log.gz 2026-02-18T07:11:57.313 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58909.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62434.log 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29465.log 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39782.log 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39861.log.gz 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58909.log.gz 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.29465.log: 88.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.62434.log.gz 2026-02-18T07:11:57.314 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44895.log 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29465.log.gz 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41408.log 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60541.log 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39782.log.gz 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25315.log 2026-02-18T07:11:57.315 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54667.log 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41408.log.gz 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60541.log.gz 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66785.log 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44895.log.gz 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25315.log.gz 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52053.log 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54667.log.gz 2026-02-18T07:11:57.316 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52515.log 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48391.log 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66785.log.gz 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52053.log.gz 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52824.log 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52515.log.gz 2026-02-18T07:11:57.317 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48391.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33437.log 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64147.log 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48391.log.gz 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.33437.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35949.log 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.33437.log.gz 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65339.log 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30604.log 2026-02-18T07:11:57.318 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64147.log.gz 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52824.log.gz 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65339.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51333.log 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65339.log.gz 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30604.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68647.log 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30604.log.gz 2026-02-18T07:11:57.319 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51333.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49240.log 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35949.log.gz 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51333.log.gz 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68647.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38915.log 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68647.log.gz 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46324.log 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36887.log.gz 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.38915.log: -- replaced with /var/log/ceph/cluster2-client.mirror.49240.log.gz 2026-02-18T07:11:57.320 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53620.log 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51945.log 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46324.log.gz 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67601.log 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53620.log.gz 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51945.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61193.log 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67601.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59456.log 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61193.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67601.log.gz 2026-02-18T07:11:57.321 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38213.log 2026-02-18T07:11:57.322 INFO:teuthology.orchestra.run.trial051.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61193.log.gz 2026-02-18T07:11:57.322 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44037.log 2026-02-18T07:11:57.322 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51945.log.gz 2026-02-18T07:11:57.322 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55445.log 2026-02-18T07:11:57.322 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67395.log 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59456.log.gz 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55445.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45003.log 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44037.log.gz 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.67395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47718.log 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55445.log.gz 2026-02-18T07:11:57.323 INFO:teuthology.orchestra.run.trial051.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.67395.log.gz 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57999.log 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45003.log.gz/var/log/ceph/cluster2-client.mirror.47718.log: 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38485.log 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47718.log.gz 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42918.log 2026-02-18T07:11:57.324 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61441.log 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57999.log.gz 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42918.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.38485.log.gz 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67478.log 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60878.log 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42918.log.gz 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61441.log.gz 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38213.log.gz 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59415.log 2026-02-18T07:11:57.325 INFO:teuthology.orchestra.run.trial051.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.60878.log: -- replaced with /var/log/ceph/cluster2-client.mirror.67478.log.gz 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50363.log 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60878.log.gz 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60811.log 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59415.log.gz 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66480.log 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54118.log 2026-02-18T07:11:57.326 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50363.log.gz 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60811.log.gz 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39249.log 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66480.log.gz 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36356.log 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54118.log.gz 2026-02-18T07:11:57.327 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60946.log 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39249.log.gz 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.36356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38786.log 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.36356.log.gz 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.35251.log 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60946.log.gz 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31240.log 2026-02-18T07:11:57.328 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.3.35251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36547.log 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38299.log 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31240.log.gz 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38786.log.gz 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41529.log 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38299.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36547.log.gz 2026-02-18T07:11:57.329 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59890.log 2026-02-18T07:11:57.330 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41529.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35754.log 2026-02-18T07:11:57.330 INFO:teuthology.orchestra.run.trial051.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.49749.log.gz 2026-02-18T07:11:57.330 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32626.log 2026-02-18T07:11:57.330 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45144.log 2026-02-18T07:11:57.331 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41701.log 90.2% 2026-02-18T07:11:57.331 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59890.log.gz 2026-02-18T07:11:57.332 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66892.log 2026-02-18T07:11:57.332 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65381.log 2026-02-18T07:11:57.332 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41701.log.gz 2026-02-18T07:11:57.332 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66892.log: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51965.log 2026-02-18T07:11:57.332 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.45144.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66892.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65381.log: gzip -5 --verbose 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38299.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr: -- /var/log/ceph/cluster2-client.mirror.57565.log 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65381.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51965.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44937.log 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57565.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35754.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64545.log 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57565.log.gz 2026-02-18T07:11:57.333 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44937.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51965.log.gz 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64723.log 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.32626.log.gz 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44937.log.gz 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47357.log 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64545.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64545.log.gz 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64723.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45999.log 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65188.log 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64723.log.gz 2026-02-18T07:11:57.334 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45999.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62206.log 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45999.log.gz 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50845.log 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47357.log.gz 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31257.log 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65188.log.gz 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62206.log.gz 2026-02-18T07:11:57.335 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33971.log 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31257.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50845.log.gz 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34662.log 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31257.log.gz 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50011.log 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44511.log 2026-02-18T07:11:57.336 INFO:teuthology.orchestra.run.trial051.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33971.log.gz 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50011.log: gzip -5 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.34662.log.gz 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.65732.log 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44511.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50011.log.gz 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40419.log 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44511.log.gz 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29927.log 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65732.log.gz 2026-02-18T07:11:57.337 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39824.log 2026-02-18T07:11:57.338 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.29927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44364.log 2026-02-18T07:11:57.338 INFO:teuthology.orchestra.run.trial051.stderr: 78.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29927.log.gz 2026-02-18T07:11:57.338 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63760.log 2026-02-18T07:11:57.338 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57606.log 2026-02-18T07:11:57.339 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.44364.log.gz/var/log/ceph/cluster2-client.mirror.63760.log: 2026-02-18T07:11:57.339 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39700.log 2026-02-18T07:11:57.339 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63760.log.gz 2026-02-18T07:11:57.339 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57606.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63095.log 2026-02-18T07:11:57.339 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52932.log 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46573.log 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39700.log.gz 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57606.log.gz 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52932.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63095.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58351.log 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.340 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52932.log.gz 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51575.log 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46573.log.gz 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53757.log 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51575.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51876.log 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51575.log.gz 2026-02-18T07:11:57.341 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.30060.log 2026-02-18T07:11:57.342 INFO:teuthology.orchestra.run.trial051.stderr: 90.1%/var/log/ceph/cluster1-client.mirror.51876.log: -- replaced with /var/log/ceph/cluster2-client.mirror.58351.log.gz 2026-02-18T07:11:57.342 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55489.log 2026-02-18T07:11:57.342 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51876.log.gz 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.3.30060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60923.log 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr: 92.1%/var/log/ceph/cluster2-client.mirror.55489.log: -- replaced with /var/log/ceph/cluster1-client.mirror.53757.log.gz 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47216.log 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60923.log: 88.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53154.log 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55489.log.gz 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60923.log.gz 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47216.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36449.log 2026-02-18T07:11:57.343 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47216.log.gz 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53154.log: gzip -5 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.30060.log.gz 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.59374.log 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53154.log.gz 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50035.log 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59374.log: gzip -5 --verbose 89.6% -- /var/log/ceph/cluster2-client.mirror.39211.log -- replaced with /var/log/ceph/cluster2-client.mirror.36449.log.gz 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59374.log.gz 2026-02-18T07:11:57.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43509.log 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50035.log.gz 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57172.log 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39211.log.gz 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43509.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60496.log 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43509.log.gz 2026-02-18T07:11:57.345 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57172.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45870.log 2026-02-18T07:11:57.346 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60496.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47150.log 2026-02-18T07:11:57.346 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60496.log.gz 2026-02-18T07:11:57.346 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44453.log 2026-02-18T07:11:57.346 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57172.log.gz 2026-02-18T07:11:57.346 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47150.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45870.log.gz 2026-02-18T07:11:57.347 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37006.log 2026-02-18T07:11:57.347 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44453.log: gzip -5 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47150.log.gz 2026-02-18T07:11:57.347 INFO:teuthology.orchestra.run.trial051.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.33997.log 2026-02-18T07:11:57.347 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46501.log 2026-02-18T07:11:57.347 INFO:teuthology.orchestra.run.trial051.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37006.log.gz 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33997.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44412.log 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33997.log.gz 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46501.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44453.log.gz 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46501.log.gz 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51430.log 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr: 87.9%/var/log/ceph/cluster2.audit.log: -- replaced with /var/log/ceph/cluster1-client.mirror.44412.log.gz 2026-02-18T07:11:57.348 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52981.log 2026-02-18T07:11:57.349 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58082.log 2026-02-18T07:11:57.349 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51430.log.gz 2026-02-18T07:11:57.349 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52981.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45027.log 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr: 87.7%/var/log/ceph/cluster2-client.mirror.58082.log: -- replaced with /var/log/ceph/cluster1-client.mirror.52981.log.gz 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29964.log 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58082.log.gz 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64422.log 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45027.log.gz 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.29964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61238.log 2026-02-18T07:11:57.350 INFO:teuthology.orchestra.run.trial051.stderr: 93.3% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43741.log 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61238.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64422.log.gz 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60248.log 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61238.log.gz 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61868.log 2026-02-18T07:11:57.351 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54508.log 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60248.log.gz 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25523.log 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61868.log.gz 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40676.log 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25523.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1-client.mirror.2.30007.log 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.25523.log.gz 2026-02-18T07:11:57.352 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54508.log.gz 2026-02-18T07:11:57.353 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40676.log: gzip -5 79.8% --verbose -- /var/log/ceph/cluster1-client.mirror.47517.log 2026-02-18T07:11:57.353 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40676.log.gz 2026-02-18T07:11:57.353 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.30007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51062.log 2026-02-18T07:11:57.353 INFO:teuthology.orchestra.run.trial006.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-18T07:11:57.354 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37564.log 2026-02-18T07:11:57.354 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47517.log.gz 2026-02-18T07:11:57.354 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51062.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30536.log 2026-02-18T07:11:57.354 INFO:teuthology.orchestra.run.trial006.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59291.log 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51062.log.gz 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.30536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56406.log 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59291.log: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.30536.log.gz 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38705.log 2026-02-18T07:11:57.355 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59291.log.gz 2026-02-18T07:11:57.356 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56406.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64319.log 2026-02-18T07:11:57.356 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38705.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56406.log.gz 2026-02-18T07:11:57.356 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38705.log.gz 2026-02-18T07:11:57.356 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58269.log 2026-02-18T07:11:57.357 INFO:teuthology.orchestra.run.trial051.stderr:gzip/var/log/ceph/cluster2-client.mirror.64319.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58393.log 2026-02-18T07:11:57.357 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64319.log.gz 2026-02-18T07:11:57.357 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58269.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58269.log.gz 2026-02-18T07:11:57.357 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37564.log.gz 2026-02-18T07:11:57.358 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57379.log 2026-02-18T07:11:57.358 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66583.log 2026-02-18T07:11:57.358 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58393.log.gz 2026-02-18T07:11:57.358 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42008.log 2026-02-18T07:11:57.359 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57379.log.gz 2026-02-18T07:11:57.359 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49455.log 2026-02-18T07:11:57.359 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66583.log: /var/log/ceph/cluster2-client.mirror.42008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64504.log 2026-02-18T07:11:57.359 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42008.log.gz 2026-02-18T07:11:57.359 INFO:teuthology.orchestra.run.trial051.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66583.log.gz 2026-02-18T07:11:57.360 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.30160.log 2026-02-18T07:11:57.360 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49455.log.gz 2026-02-18T07:11:57.360 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54190.log 2026-02-18T07:11:57.360 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64504.log: /var/log/ceph/cluster2-client.mirror.1.30160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34746.log 2026-02-18T07:11:57.360 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64504.log.gz 2026-02-18T07:11:57.361 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54190.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49505.log 2026-02-18T07:11:57.361 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54190.log.gz 2026-02-18T07:11:57.361 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34746.log: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.1.30160.log.gz 2026-02-18T07:11:57.364 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34746.log.gz 2026-02-18T07:11:57.365 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42229.log 2026-02-18T07:11:57.365 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39844.log 2026-02-18T07:11:57.365 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49505.log.gz 2026-02-18T07:11:57.365 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52467.log 2026-02-18T07:11:57.366 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42229.log.gz 2026-02-18T07:11:57.366 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49178.log 2026-02-18T07:11:57.366 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39844.log: /var/log/ceph/cluster1-client.mirror.52467.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39844.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62862.log 2026-02-18T07:11:57.366 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.366 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52467.log.gz 2026-02-18T07:11:57.367 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49178.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31031.log 2026-02-18T07:11:57.367 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62862.log: 83.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62862.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.52378.log 2026-02-18T07:11:57.367 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49178.log.gz 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.31031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43384.log 80.0% -- replaced with /var/log/ceph/cluster2-client.admin.31031.log.gz 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.0.52378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57710.log 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46684.log 2026-02-18T07:11:57.368 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43384.log: /var/log/ceph/cluster2-client.mirror.57710.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43384.log.gz 2026-02-18T07:11:57.369 INFO:teuthology.orchestra.run.trial051.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36523.log 2026-02-18T07:11:57.369 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.57710.log.gz 2026-02-18T07:11:57.369 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46684.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46684.log.gz 2026-02-18T07:11:57.370 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59250.log 2026-02-18T07:11:57.370 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52214.log 2026-02-18T07:11:57.370 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36523.log: /var/log/ceph/cluster2-client.mirror.59250.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.36523.log.gz 2026-02-18T07:11:57.371 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59250.log.gz 2026-02-18T07:11:57.371 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62275.log 2026-02-18T07:11:57.371 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62347.log 2026-02-18T07:11:57.372 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52214.log: /var/log/ceph/cluster1-client.mirror.62275.log: 41.0% -- replaced with /var/log/ceph/cluster1-client.mirror.62275.log.gz 2026-02-18T07:11:57.372 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52214.log.gz 2026-02-18T07:11:57.373 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59642.log 2026-02-18T07:11:57.373 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40634.log 2026-02-18T07:11:57.373 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62347.log.gz 2026-02-18T07:11:57.374 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59642.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59642.log.gz 2026-02-18T07:11:57.374 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38256.log 2026-02-18T07:11:57.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48573.log 2026-02-18T07:11:57.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40634.log: /var/log/ceph/cluster1-client.mirror.38256.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40634.log.gz 2026-02-18T07:11:57.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.34425.log 2026-02-18T07:11:57.376 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48573.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48573.log.gz 2026-02-18T07:11:57.376 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51405.log 2026-02-18T07:11:57.378 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.34425.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.34425.log.gz 2026-02-18T07:11:57.378 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62663.log 2026-02-18T07:11:57.379 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38256.log.gz/var/log/ceph/cluster1-client.mirror.51405.log: 2026-02-18T07:11:57.379 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51405.log.gz 2026-02-18T07:11:57.379 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61171.log 2026-02-18T07:11:57.380 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67227.log 2026-02-18T07:11:57.380 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62663.log: /var/log/ceph/cluster1-client.mirror.61171.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62663.log.gz 2026-02-18T07:11:57.380 INFO:teuthology.orchestra.run.trial051.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61171.log.gz 2026-02-18T07:11:57.380 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64381.log 2026-02-18T07:11:57.381 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47399.log 2026-02-18T07:11:57.381 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67227.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67227.log.gz 2026-02-18T07:11:57.381 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64381.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64381.log.gz 2026-02-18T07:11:57.381 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.52378.log.gz 2026-02-18T07:11:57.382 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44778.log 2026-02-18T07:11:57.382 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33896.log 2026-02-18T07:11:57.382 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47399.log.gz 2026-02-18T07:11:57.383 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37455.log 2026-02-18T07:11:57.383 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44778.log.gz 2026-02-18T07:11:57.383 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52758.log 2026-02-18T07:11:57.383 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.33896.log: /var/log/ceph/cluster1-client.mirror.37455.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.33896.log.gz 2026-02-18T07:11:57.386 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37455.log.gz 2026-02-18T07:11:57.387 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45525.log 2026-02-18T07:11:57.387 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52758.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55652.log 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52758.log.gz 2026-02-18T07:11:57.387 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38425.log 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45525.log.gz 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55652.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65645.log 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35544.log 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38425.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.30007.log.gz 2026-02-18T07:11:57.388 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38425.log.gz 2026-02-18T07:11:57.389 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65645.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55652.log.gz 2026-02-18T07:11:57.389 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57524.log 2026-02-18T07:11:57.389 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65645.log.gz/var/log/ceph/cluster2-client.mirror.35544.log: 2026-02-18T07:11:57.389 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51623.log 2026-02-18T07:11:57.389 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57524.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35544.log.gz 2026-02-18T07:11:57.390 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57524.log.gz 2026-02-18T07:11:57.390 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51623.log: gzip -5 87.7% --verbose -- /var/log/ceph/cluster1-client.mirror.37054.log 2026-02-18T07:11:57.390 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.51623.log.gz 2026-02-18T07:11:57.390 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47967.log 2026-02-18T07:11:57.391 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59146.log 2026-02-18T07:11:57.391 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.37054.log.gz 2026-02-18T07:11:57.391 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47967.log: gzip -5 --verbose -- 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47967.log.gz 2026-02-18T07:11:57.391 INFO:teuthology.orchestra.run.trial051.stderr: /var/log/ceph/cluster1-client.mirror.36132.log 2026-02-18T07:11:57.392 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62183.log 2026-02-18T07:11:57.392 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51454.log 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.36132.log.gz 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62183.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59146.log.gz 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62183.log.gz 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59229.log 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.37886.log 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51454.log.gz 2026-02-18T07:11:57.393 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53865.log 2026-02-18T07:11:57.394 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59229.log.gz 2026-02-18T07:11:57.394 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.37886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36679.log 2026-02-18T07:11:57.394 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34861.log 2026-02-18T07:11:57.394 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53865.log.gz 2026-02-18T07:11:57.395 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54443.log 2026-02-18T07:11:57.395 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.34861.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.34861.log.gz 2026-02-18T07:11:57.395 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.0.37886.log.gz 2026-02-18T07:11:57.395 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45621.log 2026-02-18T07:11:57.396 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54443.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35666.log 2026-02-18T07:11:57.396 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54443.log.gz 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45621.log: gzip -5 --verbose -- 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45621.log.gz 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr: /var/log/ceph/cluster1-client.mirror.47838.log 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44266.log 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35666.log.gz 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47838.log: gzip -5 --verbose -- 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47838.log.gz 2026-02-18T07:11:57.397 INFO:teuthology.orchestra.run.trial051.stderr: /var/log/ceph/cluster1-client.mirror.48501.log 2026-02-18T07:11:57.398 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59932.log 2026-02-18T07:11:57.398 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36679.log.gz 2026-02-18T07:11:57.398 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48501.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44266.log.gz 2026-02-18T07:11:57.398 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48501.log.gz 2026-02-18T07:11:57.398 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40893.log 2026-02-18T07:11:57.399 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59932.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49136.log 2026-02-18T07:11:57.399 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59932.log.gz 2026-02-18T07:11:57.399 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44340.log 2026-02-18T07:11:57.399 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41862.log 2026-02-18T07:11:57.400 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44340.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44340.log.gz 2026-02-18T07:11:57.400 INFO:teuthology.orchestra.run.trial051.stderr:gzip 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49136.log.gz 2026-02-18T07:11:57.400 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.34340.log 2026-02-18T07:11:57.401 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36571.log 2026-02-18T07:11:57.401 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.34340.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41862.log.gz 2026-02-18T07:11:57.401 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.34340.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68255.log 2026-02-18T07:11:57.401 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.401 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36571.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36571.log.gz 2026-02-18T07:11:57.402 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64565.log 2026-02-18T07:11:57.402 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68255.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68255.log.gz 2026-02-18T07:11:57.402 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47128.log 2026-02-18T07:11:57.403 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64565.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32971.log 2026-02-18T07:11:57.403 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47128.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64565.log.gz 2026-02-18T07:11:57.404 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.47128.log.gz 2026-02-18T07:11:57.404 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62327.log 2026-02-18T07:11:57.405 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59208.log 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32971.log.gz 2026-02-18T07:11:57.405 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.405 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38127.log 2026-02-18T07:11:57.405 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59208.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62327.log.gz 2026-02-18T07:11:57.406 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57834.log 2026-02-18T07:11:57.406 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56113.log 2026-02-18T07:11:57.406 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59208.log.gz 2026-02-18T07:11:57.407 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57834.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57834.log.gz 2026-02-18T07:11:57.407 INFO:teuthology.orchestra.run.trial051.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29964.log.gz 2026-02-18T07:11:57.408 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35871.log 2026-02-18T07:11:57.408 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56113.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45099.log 2026-02-18T07:11:57.409 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.35871.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.35871.log.gz 2026-02-18T07:11:57.409 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56113.log.gz 2026-02-18T07:11:57.409 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38127.log.gz 2026-02-18T07:11:57.409 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58620.log 2026-02-18T07:11:57.410 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45099.log: gzip -5 --verbose -- 87.8% /var/log/ceph/cluster2-client.mirror.58455.log 2026-02-18T07:11:57.410 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.45099.log.gz 2026-02-18T07:11:57.410 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67621.log 2026-02-18T07:11:57.410 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58620.log.gz 2026-02-18T07:11:57.410 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58455.log: gzip 87.9% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43777.log 2026-02-18T07:11:57.411 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58455.log.gz 2026-02-18T07:11:57.411 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67621.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33084.log 2026-02-18T07:11:57.411 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67621.log.gz 2026-02-18T07:11:57.411 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43777.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43777.log.gz 2026-02-18T07:11:57.411 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50988.log 2026-02-18T07:11:57.412 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51987.log 2026-02-18T07:11:57.412 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66871.log 2026-02-18T07:11:57.412 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44475.log 2026-02-18T07:11:57.413 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51987.log.gz 2026-02-18T07:11:57.413 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66871.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50988.log.gz 2026-02-18T07:11:57.413 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66871.log.gz 2026-02-18T07:11:57.413 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35731.log 2026-02-18T07:11:57.413 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42330.log 2026-02-18T07:11:57.414 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35731.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44475.log.gz 2026-02-18T07:11:57.414 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34093.log 2026-02-18T07:11:57.414 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67121.log 2026-02-18T07:11:57.415 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.33084.log.gz 2026-02-18T07:11:57.415 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34093.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34093.log.gz 2026-02-18T07:11:57.416 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.35731.log.gz 2026-02-18T07:11:57.417 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.36887.log 2026-02-18T07:11:57.417 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67121.log: gzip -5 --verbose -- 86.9% /var/log/ceph/cluster2-client.admin.28735.log 2026-02-18T07:11:57.417 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.67121.log.gz 2026-02-18T07:11:57.418 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.0.36887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57503.log 2026-02-18T07:11:57.418 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28735.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28735.log.gz 2026-02-18T07:11:57.420 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33621.log 2026-02-18T07:11:57.420 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57503.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57503.log.gz 2026-02-18T07:11:57.420 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56643.log 2026-02-18T07:11:57.421 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.39515.log 2026-02-18T07:11:57.421 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33621.log.gz 2026-02-18T07:11:57.421 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56643.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56643.log.gz 2026-02-18T07:11:57.421 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33525.log 2026-02-18T07:11:57.422 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.39515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66270.log 2026-02-18T07:11:57.422 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33525.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.33525.log.gz 2026-02-18T07:11:57.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.39515.log.gz 2026-02-18T07:11:57.438 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.32151.log 2026-02-18T07:11:57.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66270.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66270.log.gz 2026-02-18T07:11:57.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38725.log 2026-02-18T07:11:57.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.32151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50122.log 2026-02-18T07:11:57.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.32151.log.gz 2026-02-18T07:11:57.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36409.log 2026-02-18T07:11:57.440 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38725.log.gz 2026-02-18T07:11:57.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29710.log 2026-02-18T07:11:57.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36409.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50122.log.gz 2026-02-18T07:11:57.441 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36409.log.gz 2026-02-18T07:11:57.441 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38645.log 2026-02-18T07:11:57.441 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29710.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29710.log.gz 2026-02-18T07:11:57.441 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55081.log 2026-02-18T07:11:57.441 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61688.log 2026-02-18T07:11:57.442 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38645.log.gz 2026-02-18T07:11:57.442 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42288.log 2026-02-18T07:11:57.442 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61688.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55081.log.gz 2026-02-18T07:11:57.442 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61688.log.gz 2026-02-18T07:11:57.442 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40577.log 2026-02-18T07:11:57.443 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.29964.log 2026-02-18T07:11:57.443 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40577.log: /var/log/ceph/cluster2-client.mirror.42288.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42288.log.gz 2026-02-18T07:11:57.444 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25744.log 2026-02-18T07:11:57.453 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.29964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57813.log 2026-02-18T07:11:57.458 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25744.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25744.log.gz 2026-02-18T07:11:57.466 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48790.log 2026-02-18T07:11:57.466 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57813.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57813.log.gz 2026-02-18T07:11:57.473 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44977.log 2026-02-18T07:11:57.475 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48790.log: 87.7% 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.36887.log.gz 2026-02-18T07:11:57.475 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.48790.log.gz 2026-02-18T07:11:57.476 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36777.log 2026-02-18T07:11:57.476 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44977.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44977.log.gz 2026-02-18T07:11:57.477 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35111.log 2026-02-18T07:11:57.477 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.36777.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.36777.log.gz 2026-02-18T07:11:57.478 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45186.log 2026-02-18T07:11:57.481 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35111.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.35111.log.gz 2026-02-18T07:11:57.481 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39228.log 2026-02-18T07:11:57.482 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45186.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45186.log.gz 2026-02-18T07:11:57.482 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45300.log 2026-02-18T07:11:57.483 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39228.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39228.log.gz 2026-02-18T07:11:57.483 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56732.log 2026-02-18T07:11:57.484 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45300.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45300.log.gz 2026-02-18T07:11:57.485 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33709.log 2026-02-18T07:11:57.486 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56732.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56732.log.gz 2026-02-18T07:11:57.486 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34703.log 2026-02-18T07:11:57.487 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33709.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.29964.log.gz 2026-02-18T07:11:57.488 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29416.log 2026-02-18T07:11:57.489 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34703.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33709.log.gz 2026-02-18T07:11:57.490 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53668.log 2026-02-18T07:11:57.490 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.29416.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29416.log.gz 2026-02-18T07:11:57.491 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46252.log 2026-02-18T07:11:57.491 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53668.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53668.log.gz 2026-02-18T07:11:57.492 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37627.log 2026-02-18T07:11:57.492 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46252.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34703.log.gz 2026-02-18T07:11:57.492 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46252.log.gz 2026-02-18T07:11:57.492 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67740.log 2026-02-18T07:11:57.492 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41845.log 2026-02-18T07:11:57.493 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37627.log: /var/log/ceph/cluster1-client.mirror.67740.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67740.log.gz 2026-02-18T07:11:57.493 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61508.log 2026-02-18T07:11:57.493 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41845.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41845.log.gz 2026-02-18T07:11:57.494 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44728.log 2026-02-18T07:11:57.494 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61508.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61508.log.gz 2026-02-18T07:11:57.494 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53423.log 2026-02-18T07:11:57.495 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44728.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44728.log.gz 2026-02-18T07:11:57.496 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64702.log 2026-02-18T07:11:57.496 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37627.log.gz 2026-02-18T07:11:57.497 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53423.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59828.log 2026-02-18T07:11:57.497 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53423.log.gz 2026-02-18T07:11:57.497 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64702.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64702.log.gz 2026-02-18T07:11:57.497 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38150.log 2026-02-18T07:11:57.498 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59312.log 2026-02-18T07:11:57.498 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59828.log: /var/log/ceph/cluster2-client.mirror.38150.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59828.log.gz 2026-02-18T07:11:57.499 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48621.log 2026-02-18T07:11:57.499 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59312.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59312.log.gz 2026-02-18T07:11:57.500 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66747.log 2026-02-18T07:11:57.501 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.48621.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38150.log.gz 2026-02-18T07:11:57.501 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49622.log 2026-02-18T07:11:57.502 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66747.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66747.log.gz 2026-02-18T07:11:57.502 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37123.log 2026-02-18T07:11:57.503 INFO:teuthology.orchestra.run.trial051.stderr: 92.5%/var/log/ceph/cluster2-client.mirror.49622.log: -- replaced with /var/log/ceph/cluster1-client.mirror.3.35251.log.gz 2026-02-18T07:11:57.503 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.43295.log 2026-02-18T07:11:57.504 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37123.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49622.log.gz 2026-02-18T07:11:57.504 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57627.log 2026-02-18T07:11:57.507 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.43295.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37123.log.gz 2026-02-18T07:11:57.507 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42859.log 2026-02-18T07:11:57.508 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57627.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57627.log.gz 2026-02-18T07:11:57.508 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59580.log 2026-02-18T07:11:57.511 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42859.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48621.log.gz 2026-02-18T07:11:57.511 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61643.log 2026-02-18T07:11:57.512 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59580.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63051.log.gz 2026-02-18T07:11:57.513 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59580.log.gz 2026-02-18T07:11:57.513 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63528.log 2026-02-18T07:11:57.513 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61643.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60136.log 2026-02-18T07:11:57.513 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61643.log.gz 2026-02-18T07:11:57.514 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63528.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63528.log.gz 2026-02-18T07:11:57.526 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32670.log 2026-02-18T07:11:57.526 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54688.log 2026-02-18T07:11:57.527 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60136.log.gz 2026-02-18T07:11:57.527 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55611.log 2026-02-18T07:11:57.528 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54688.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54688.log.gz 2026-02-18T07:11:57.529 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38856.log 2026-02-18T07:11:57.529 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55611.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.55611.log.gz 2026-02-18T07:11:57.530 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40913.log 2026-02-18T07:11:57.530 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38856.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38856.log.gz 2026-02-18T07:11:57.531 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62310.log 2026-02-18T07:11:57.531 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40913.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40913.log.gz 2026-02-18T07:11:57.532 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57689.log 2026-02-18T07:11:57.532 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62310.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62310.log.gz 2026-02-18T07:11:57.532 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55262.log 2026-02-18T07:11:57.533 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57689.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57689.log.gz 2026-02-18T07:11:57.533 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-18T07:11:57.534 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55262.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55262.log.gz 2026-02-18T07:11:57.534 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66348.log 2026-02-18T07:11:57.543 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43974.log 2026-02-18T07:11:57.556 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66348.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66348.log.gz 2026-02-18T07:11:57.558 INFO:teuthology.orchestra.run.trial051.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.43295.log.gz 2026-02-18T07:11:57.559 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40693.log 2026-02-18T07:11:57.559 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.43974.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43974.log.gz 2026-02-18T07:11:57.560 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56155.log 2026-02-18T07:11:57.560 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40693.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40693.log.gz 2026-02-18T07:11:57.561 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28395.log 2026-02-18T07:11:57.562 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56155.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56155.log.gz 2026-02-18T07:11:57.562 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51088.log 2026-02-18T07:11:57.562 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28395.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28395.log.gz 2026-02-18T07:11:57.564 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47991.log 2026-02-18T07:11:57.564 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51088.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51088.log.gz 2026-02-18T07:11:57.564 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33019.log 2026-02-18T07:11:57.565 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47991.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.47991.log.gz 2026-02-18T07:11:57.565 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55286.log 2026-02-18T07:11:57.566 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33019.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33019.log.gz 2026-02-18T07:11:57.566 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43884.log 2026-02-18T07:11:57.566 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55286.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55286.log.gz 2026-02-18T07:11:57.567 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60968.log 2026-02-18T07:11:57.568 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.43884.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43884.log.gz 2026-02-18T07:11:57.570 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48280.log 2026-02-18T07:11:57.570 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60968.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60968.log.gz 2026-02-18T07:11:57.570 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55007.log 2026-02-18T07:11:57.571 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48280.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48280.log.gz 2026-02-18T07:11:57.571 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59684.log 2026-02-18T07:11:57.571 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55007.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55007.log.gz 2026-02-18T07:11:57.573 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.56293.log 2026-02-18T07:11:57.573 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59684.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59684.log.gz 2026-02-18T07:11:57.573 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55423.log 2026-02-18T07:11:57.581 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.0.56293.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40253.log.gz 2026-02-18T07:11:57.582 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56904.log 2026-02-18T07:11:57.583 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55423.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.55423.log.gz 2026-02-18T07:11:57.584 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66624.log 2026-02-18T07:11:57.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56904.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56904.log.gz 2026-02-18T07:11:57.584 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35070.log 2026-02-18T07:11:57.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66624.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66624.log.gz 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67860.log 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35070.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.56293.log.gz 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.35070.log.gz 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35774.log 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.67860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40039.log 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.67860.log.gz 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43193.log 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40039.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40039.log.gz 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40019.log 2026-02-18T07:11:57.588 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43193.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43193.log.gz 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56758.log 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40019.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40019.log.gz 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34887.log 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56758.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56758.log.gz 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48453.log 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34887.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.35774.log.gz 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34887.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59353.log 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.592 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45911.log 2026-02-18T07:11:57.593 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48453.log.gz 2026-02-18T07:11:57.593 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59353.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60406.log 2026-02-18T07:11:57.593 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45911.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59353.log.gz 2026-02-18T07:11:57.593 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58868.log 2026-02-18T07:11:57.596 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60406.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60406.log.gz 2026-02-18T07:11:57.596 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53937.log 2026-02-18T07:11:57.596 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45911.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47336.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53937.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58868.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53937.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57358.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.47336.log: -- /var/log/ceph/cluster1-client.mirror.44802.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47336.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57358.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64745.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44802.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44802.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57358.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42308.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64745.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50154.log 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.64745.log.gz 2026-02-18T07:11:57.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42308.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42308.log.gz 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34542.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50154.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66415.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34542.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50154.log.gz 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.34542.log.gz 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49423.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50628.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66415.log: /var/log/ceph/cluster2-client.mirror.49423.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66415.log.gz 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49423.log.gz 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45642.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40298.log 2026-02-18T07:11:57.600 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50628.log: /var/log/ceph/cluster2-client.mirror.45642.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50628.log.gz 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48549.log 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45642.log.gz 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63427.log 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48549.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40298.log.gz 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48549.log.gz 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63116.log 2026-02-18T07:11:57.602 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.63427.log: /var/log/ceph/cluster1-client.mirror.46842.log 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63116.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63116.log.gz 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63427.log.gz 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29808.log 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55129.log 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46842.log.gz 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29808.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29808.log.gz 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54933.log 2026-02-18T07:11:57.604 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55129.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55129.log.gz 2026-02-18T07:11:57.605 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34482.log 2026-02-18T07:11:57.605 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32861.log 2026-02-18T07:11:57.605 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34482.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34482.log.gz 2026-02-18T07:11:57.605 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54933.log.gz 2026-02-18T07:11:57.606 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58331.log 2026-02-18T07:11:57.606 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32439.log 2026-02-18T07:11:57.607 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58331.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32861.log.gz 2026-02-18T07:11:57.607 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58331.log.gz 2026-02-18T07:11:57.607 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56462.log 2026-02-18T07:11:57.607 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32439.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32439.log.gz 2026-02-18T07:11:57.608 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44874.log 2026-02-18T07:11:57.608 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56462.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40871.log 2026-02-18T07:11:57.608 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56462.log.gz 2026-02-18T07:11:57.609 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34138.log 2026-02-18T07:11:57.609 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40871.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44874.log.gz 2026-02-18T07:11:57.609 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40871.log.gz 2026-02-18T07:11:57.609 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65910.log 2026-02-18T07:11:57.609 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52306.log 2026-02-18T07:11:57.610 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65910.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65910.log.gz 2026-02-18T07:11:57.612 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34138.log.gz 2026-02-18T07:11:57.613 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66686.log 2026-02-18T07:11:57.613 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37779.log 2026-02-18T07:11:57.614 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52306.log: /var/log/ceph/cluster2-client.mirror.66686.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52306.log.gz 2026-02-18T07:11:57.614 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66686.log.gz 2026-02-18T07:11:57.614 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40972.log 2026-02-18T07:11:57.614 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46866.log 2026-02-18T07:11:57.615 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40972.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64645.log.gz 2026-02-18T07:11:57.616 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42271.log 2026-02-18T07:11:57.617 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46866.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46866.log.gz 2026-02-18T07:11:57.617 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37779.log.gz 2026-02-18T07:11:57.617 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57586.log 2026-02-18T07:11:57.618 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58827.log 2026-02-18T07:11:57.618 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57586.log: /var/log/ceph/cluster2-client.mirror.42271.log: 79.8% 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57586.log.gz 2026-02-18T07:11:57.618 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.42271.log.gz 2026-02-18T07:11:57.619 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53841.log 2026-02-18T07:11:57.619 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32947.log 2026-02-18T07:11:57.619 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58827.log.gz 2026-02-18T07:11:57.619 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43840.log 2026-02-18T07:11:57.620 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32947.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53841.log.gz 2026-02-18T07:11:57.620 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.32947.log.gz 2026-02-18T07:11:57.620 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30553.log 2026-02-18T07:11:57.620 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61913.log 2026-02-18T07:11:57.622 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.43840.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43840.log.gz 2026-02-18T07:11:57.622 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.30553.log: 70.4% -- replaced with /var/log/ceph/cluster1-client.admin.30553.log.gz 2026-02-18T07:11:57.622 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34313.log 2026-02-18T07:11:57.622 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54840.log 2026-02-18T07:11:57.622 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61913.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.34313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61283.log 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54840.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.34313.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54840.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53050.log 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43235.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66805.log/var/log/ceph/cluster1-client.mirror.61283.log: 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61283.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44433.log 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40656.log.gz 2026-02-18T07:11:57.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41132.log 2026-02-18T07:11:57.625 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66805.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44433.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42411.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40713.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53050.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66665.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44433.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40713.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42411.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40713.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43719.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57007.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66665.log.gz 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55970.log 2026-02-18T07:11:57.627 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43719.log.gz 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43421.log 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57007.log: /var/log/ceph/cluster2-client.mirror.55970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62413.log 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57007.log.gz 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43421.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55970.log.gz 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61531.log 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43421.log.gz 2026-02-18T07:11:57.628 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48995.log 2026-02-18T07:11:57.629 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62413.log: /var/log/ceph/cluster1-client.mirror.61531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60181.log 2026-02-18T07:11:57.629 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61531.log.gz 2026-02-18T07:11:57.629 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.62413.log.gz 2026-02-18T07:11:57.629 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63572.log 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48995.log.gz 2026-02-18T07:11:57.630 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.630 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38745.log 2026-02-18T07:11:57.630 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60181.log.gz 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63572.log: 93.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59126.log 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40419.log.gz 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38745.log: gzip -5 --verbose 88.5% -- /var/log/ceph/cluster2-client.mirror.49937.log 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.63572.log.gz 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59126.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38745.log.gz 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64089.log 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59126.log.gz 2026-02-18T07:11:57.631 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39960.log 2026-02-18T07:11:57.632 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49937.log.gz 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64089.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39463.log 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41371.log.gz 87.8% 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64089.log.gz 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42486.log 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39960.log.gz 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52846.log 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: 93.7%/var/log/ceph/cluster2-client.mirror.42486.log: -- replaced with /var/log/ceph/cluster2-client.mirror.43741.log.gz 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42486.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41783.log 2026-02-18T07:11:57.633 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58248.log 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41783.log: gzip -5 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52846.log.gz 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.32513.log 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58248.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41783.log.gz 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57131.log 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58248.log.gz 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62540.log 2026-02-18T07:11:57.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57131.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.32513.log.gz 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52614.log 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57131.log.gz 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66034.log 87.0% 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62540.log.gz 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44609.log 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66034.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52614.log.gz 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57420.log 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66034.log.gz 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39742.log 2026-02-18T07:11:57.636 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44609.log.gz 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57420.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30655.log 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66392.log 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30655.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57420.log.gz 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr: 64.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30655.log.gz 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53291.log 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40518.log 2026-02-18T07:11:57.638 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66392.log.gz 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37650.log 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40518.log: 79.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41487.log 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40518.log.gz 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57958.log 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53291.log.gz 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65150.log 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61103.log 2026-02-18T07:11:57.639 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41487.log.gz 2026-02-18T07:11:57.640 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57958.log: /var/log/ceph/cluster2-client.mirror.65150.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57958.log.gz 2026-02-18T07:11:57.640 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62369.log 2026-02-18T07:11:57.640 INFO:teuthology.orchestra.run.trial051.stderr: 90.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55740.log 2026-02-18T07:11:57.640 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.65150.log.gz 2026-02-18T07:11:57.641 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61103.log: /var/log/ceph/cluster2-client.mirror.62369.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61103.log.gz 2026-02-18T07:11:57.641 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35642.log 2026-02-18T07:11:57.641 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50204.log 2026-02-18T07:11:57.641 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55740.log.gz 2026-02-18T07:11:57.641 INFO:teuthology.orchestra.run.trial051.stderr: 93.6%/var/log/ceph/cluster1-client.mirror.35642.log: -- replaced with /var/log/ceph/cluster2-client.mirror.37650.log.gz 2026-02-18T07:11:57.642 INFO:teuthology.orchestra.run.trial051.stderr: 88.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64275.log 2026-02-18T07:11:57.642 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.35642.log.gz 2026-02-18T07:11:57.642 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65130.log 2026-02-18T07:11:57.642 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31218.log 2026-02-18T07:11:57.642 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50204.log.gz 2026-02-18T07:11:57.643 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.64275.log.gz 2026-02-18T07:11:57.643 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56382.log 2026-02-18T07:11:57.643 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65130.log.gz 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31218.log: gzip 53.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31218.log.gz 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64831.log 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56382.log: gzip -5 --verbose 87.6% -- /var/log/ceph/cluster1-client.mirror.30689.log 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.56382.log.gz 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-18T07:11:57.644 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30689.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64831.log.gz 2026-02-18T07:11:57.645 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33064.log 2026-02-18T07:11:57.645 INFO:teuthology.orchestra.run.trial051.stderr: 54.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30689.log.gz 2026-02-18T07:11:57.645 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62292.log 2026-02-18T07:11:57.645 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46393.log 2026-02-18T07:11:57.645 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33064.log: /var/log/ceph/cluster2-client.mirror.62292.log: 39.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62292.log.gz 2026-02-18T07:11:57.648 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33064.log.gz 2026-02-18T07:11:57.654 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38279.log 2026-02-18T07:11:57.654 INFO:teuthology.orchestra.run.trial051.stderr:gzip/var/log/ceph/cluster2-client.mirror.46393.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63284.log 2026-02-18T07:11:57.655 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38279.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46393.log.gz 2026-02-18T07:11:57.656 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60091.log 2026-02-18T07:11:57.656 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63284.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63284.log.gz 2026-02-18T07:11:57.656 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45393.log 2026-02-18T07:11:57.657 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60091.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60091.log.gz 2026-02-18T07:11:57.657 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49401.log 2026-02-18T07:11:57.658 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45393.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38279.log.gz 2026-02-18T07:11:57.659 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45393.log.gz 2026-02-18T07:11:57.667 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56503.log 2026-02-18T07:11:57.667 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42837.log 2026-02-18T07:11:57.668 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49401.log: /var/log/ceph/cluster1-client.mirror.56503.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49401.log.gz 2026-02-18T07:11:57.669 INFO:teuthology.orchestra.run.trial051.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56503.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57916.log 2026-02-18T07:11:57.669 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.669 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.62862.log 2026-02-18T07:11:57.669 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57916.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42837.log.gz 2026-02-18T07:11:57.670 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56966.log 2026-02-18T07:11:57.670 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.62862.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57916.log.gz 2026-02-18T07:11:57.674 INFO:teuthology.orchestra.run.trial051.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.62862.log.gz 2026-02-18T07:11:57.679 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62138.log 2026-02-18T07:11:57.679 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37670.log 2026-02-18T07:11:57.679 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56966.log.gz 2026-02-18T07:11:57.680 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62971.log 2026-02-18T07:11:57.680 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62138.log.gz 2026-02-18T07:11:57.681 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59498.log 2026-02-18T07:11:57.681 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62971.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62971.log.gz 2026-02-18T07:11:57.682 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63740.log 2026-02-18T07:11:57.682 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59498.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59498.log.gz 2026-02-18T07:11:57.683 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66957.log 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr: 93.5%/var/log/ceph/cluster2-client.mirror.63740.log: -- replaced with /var/log/ceph/cluster1-client.mirror.37670.log.gz 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63740.log.gz 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42330.log.gz 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38545.log 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66957.log: gzip -5 --verbose -- 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66957.log.gz 2026-02-18T07:11:57.684 INFO:teuthology.orchestra.run.trial051.stderr: /var/log/ceph/cluster2-client.admin.25401.log 2026-02-18T07:11:57.685 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46907.log 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38545.log.gz 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25401.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25401.log.gz 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50505.log 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45324.log 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60901.log 2026-02-18T07:11:57.686 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50505.log: /var/log/ceph/cluster1-client.mirror.45324.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50505.log.gz 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45324.log.gz 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.46907.log.gz 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48108.log 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58164.log 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60901.log.gz 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53495.log 2026-02-18T07:11:57.687 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48108.log: /var/log/ceph/cluster2-client.mirror.58164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55632.log 2026-02-18T07:11:57.688 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53495.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53495.log.gz 2026-02-18T07:11:57.689 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58164.log.gz 2026-02-18T07:11:57.689 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38525.log 2026-02-18T07:11:57.689 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48108.log.gz 2026-02-18T07:11:57.689 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55632.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55632.log.gz 2026-02-18T07:11:57.690 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63325.log 2026-02-18T07:11:57.691 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45774.log 2026-02-18T07:11:57.691 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38525.log.gz 2026-02-18T07:11:57.691 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63325.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59270.log 2026-02-18T07:11:57.691 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63325.log.gz 2026-02-18T07:11:57.692 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45774.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38342.log 2026-02-18T07:11:57.692 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59270.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45774.log.gz 2026-02-18T07:11:57.692 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49601.log 2026-02-18T07:11:57.692 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38342.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59270.log.gz 2026-02-18T07:11:57.693 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37143.log 2026-02-18T07:11:57.693 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49601.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55812.log 2026-02-18T07:11:57.693 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49601.log.gz 2026-02-18T07:11:57.694 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37756.log 2026-02-18T07:11:57.694 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55812.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55812.log.gz 2026-02-18T07:11:57.695 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39683.log 2026-02-18T07:11:57.695 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37756.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.38342.log.gz 2026-02-18T07:11:57.696 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60743.log 2026-02-18T07:11:57.696 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39683.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39683.log.gz 2026-02-18T07:11:57.696 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53737.log 2026-02-18T07:11:57.697 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60743.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60743.log.gz 2026-02-18T07:11:57.698 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44135.log 2026-02-18T07:11:57.698 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53737.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37756.log.gz 2026-02-18T07:11:57.698 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37143.log.gz 2026-02-18T07:11:57.698 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57875.log 2026-02-18T07:11:57.698 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44135.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44135.log.gz 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53737.log.gz 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60068.log 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29178.log 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster1-client.mirror.60068.log: -- replaced with /var/log/ceph/cluster2-client.mirror.57875.log.gz 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32221.log 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60068.log.gz 2026-02-18T07:11:57.700 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53594.log 2026-02-18T07:11:57.701 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29178.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29178.log.gz 2026-02-18T07:11:57.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32221.log: gzip -5 --verbose -- 84.8% /var/log/ceph/cluster2-client.mirror.43052.log 2026-02-18T07:11:57.702 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.32221.log.gz 2026-02-18T07:11:57.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49214.log 2026-02-18T07:11:57.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43052.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53594.log.gz 2026-02-18T07:11:57.702 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57276.log 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34245.log 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49214.log.gz 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57276.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57276.log.gz 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36475.log 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57544.log 2026-02-18T07:11:57.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36475.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36475.log.gz 2026-02-18T07:11:57.711 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63181.log 2026-02-18T07:11:57.712 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57544.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57544.log.gz 2026-02-18T07:11:57.712 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56092.log 2026-02-18T07:11:57.713 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63181.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63181.log.gz 2026-02-18T07:11:57.713 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64000.log 2026-02-18T07:11:57.714 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56092.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56092.log.gz 2026-02-18T07:11:57.714 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44654.log 2026-02-18T07:11:57.715 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64000.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64000.log.gz 2026-02-18T07:11:57.715 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47652.log 2026-02-18T07:11:57.717 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44654.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44654.log.gz 2026-02-18T07:11:57.717 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66075.log 2026-02-18T07:11:57.718 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47652.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47652.log.gz 2026-02-18T07:11:57.718 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61981.log 2026-02-18T07:11:57.719 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66075.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66075.log.gz 2026-02-18T07:11:57.719 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.40105.log 2026-02-18T07:11:57.720 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61981.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61981.log.gz 2026-02-18T07:11:57.721 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58102.log 2026-02-18T07:11:57.729 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.2.40105.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45846.log 2026-02-18T07:11:57.734 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58102.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.40105.log.gz 2026-02-18T07:11:57.735 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47469.log 2026-02-18T07:11:57.736 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45846.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45846.log.gz 2026-02-18T07:11:57.736 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67162.log 2026-02-18T07:11:57.737 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47469.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47469.log.gz 2026-02-18T07:11:57.737 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43213.log 2026-02-18T07:11:57.738 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67162.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67162.log.gz 2026-02-18T07:11:57.738 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47610.log 2026-02-18T07:11:57.739 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43213.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43213.log.gz 2026-02-18T07:11:57.739 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62561.log 2026-02-18T07:11:57.740 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47610.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47610.log.gz 2026-02-18T07:11:57.741 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55170.log 2026-02-18T07:11:57.741 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62561.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62561.log.gz 2026-02-18T07:11:57.742 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53128.log 2026-02-18T07:11:57.742 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55170.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58102.log.gz 2026-02-18T07:11:57.743 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66827.log.gz 2026-02-18T07:11:57.743 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55170.log.gz 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35046.log 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.53128.log: --verbose -- /var/log/ceph/cluster2-client.mirror.49664.log 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35046.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53128.log.gz 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35046.log.gz 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33463.log 2026-02-18T07:11:57.745 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49664.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65208.log 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49664.log.gz 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48328.log 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65208.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.33463.log.gz 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65208.log.gz 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50481.log 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.48328.log: -- /var/log/ceph/cluster1-client.mirror.35594.log 2026-02-18T07:11:57.747 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48328.log.gz/var/log/ceph/cluster1-client.mirror.50481.log: 2026-02-18T07:11:57.749 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50481.log.gz 2026-02-18T07:11:57.749 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54792.log 2026-02-18T07:11:57.749 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35594.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.35594.log.gz 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25879.log 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.54792.log: -- /var/log/ceph/cluster2-client.mirror.35818.log 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54792.log.gz 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.25879.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25879.log.gz 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62581.log 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40814.log 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62581.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62581.log.gz 2026-02-18T07:11:57.750 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51229.log 2026-02-18T07:11:57.759 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58641.log 2026-02-18T07:11:57.766 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45075.log 2026-02-18T07:11:57.766 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58641.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58641.log.gz 2026-02-18T07:11:57.773 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51229.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59105.log 2026-02-18T07:11:57.773 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51229.log.gz 2026-02-18T07:11:57.773 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45075.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45075.log.gz 2026-02-18T07:11:57.782 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48621.log 2026-02-18T07:11:57.782 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59105.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59105.log.gz 2026-02-18T07:11:57.791 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51271.log 2026-02-18T07:11:57.793 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48621.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41825.log.gz 2026-02-18T07:11:57.794 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61711.log 2026-02-18T07:11:57.794 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51271.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51271.log.gz 2026-02-18T07:11:57.795 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34790.log 2026-02-18T07:11:57.795 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61711.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61711.log.gz 2026-02-18T07:11:57.796 INFO:teuthology.orchestra.run.trial051.stderr: 93.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63137.log 2026-02-18T07:11:57.802 INFO:teuthology.orchestra.run.trial006.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-18T07:11:57.808 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34790.log: -- replaced with /var/log/ceph/cluster2-client.mirror.48621.log.gz 2026-02-18T07:11:57.809 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62116.log 2026-02-18T07:11:57.809 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63137.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63137.log.gz 2026-02-18T07:11:57.820 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57400.log 2026-02-18T07:11:57.820 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62116.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62116.log.gz 2026-02-18T07:11:57.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56068.log 2026-02-18T07:11:57.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57400.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57400.log.gz 2026-02-18T07:11:57.850 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42901.log 2026-02-18T07:11:57.850 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56068.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56068.log.gz 2026-02-18T07:11:57.859 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40360.log 2026-02-18T07:11:57.859 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42901.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42901.log.gz 2026-02-18T07:11:57.868 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50967.log 2026-02-18T07:11:57.873 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40360.log: 79.6% 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39083.log.gz 2026-02-18T07:11:57.875 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38064.log 2026-02-18T07:11:57.875 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50967.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50967.log.gz 2026-02-18T07:11:57.875 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37886.log 2026-02-18T07:11:57.880 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38064.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38064.log.gz 2026-02-18T07:11:57.880 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40360.log.gz 2026-02-18T07:11:57.881 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38952.log 2026-02-18T07:11:57.881 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37886.log: 84.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37886.log.gz 2026-02-18T07:11:57.881 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44180.log 2026-02-18T07:11:57.881 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38952.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38952.log.gz 2026-02-18T07:11:57.881 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41924.log 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44180.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44180.log.gz 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67560.log 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41924.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41924.log.gz 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66936.log 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67560.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67560.log.gz 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42067.log 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66936.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66936.log.gz 2026-02-18T07:11:57.885 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47240.log 2026-02-18T07:11:57.892 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42067.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41529.log.gz 2026-02-18T07:11:57.892 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53536.log 2026-02-18T07:11:57.892 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47240.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.47240.log.gz 2026-02-18T07:11:57.892 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42750.log.gz 2026-02-18T07:11:57.908 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50813.log 2026-02-18T07:11:57.908 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57193.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50813.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50813.log.gz 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53536.log.gz 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63222.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.57193.log: -- /var/log/ceph/cluster2-client.mirror.39141.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57193.log.gz 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45933.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr: 87.1%/var/log/ceph/cluster2-client.mirror.39141.log: -- replaced with /var/log/ceph/cluster2-client.mirror.63222.log.gz 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56986.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39141.log.gz 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41090.log 2026-02-18T07:11:57.931 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45933.log: /var/log/ceph/cluster2-client.mirror.56986.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45933.log.gz 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64189.log.gz 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59560.log 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56986.log.gz 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54068.log 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51160.log 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59560.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41090.log.gz 2026-02-18T07:11:57.932 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59560.log.gz 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58537.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34974.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51160.log.gz 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54068.log.gz 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55764.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34974.log.gz 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55764.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58537.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66136.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-mon.a.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55764.log.gz 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47790.log 2026-02-18T07:11:57.933 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66136.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66136.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64603.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68549.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.47790.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51900.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68549.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68549.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64603.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48232.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40498.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51900.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40397.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40498.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.48232.log.gz 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28882.log 2026-02-18T07:11:57.934 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40397.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40397.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50100.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65687.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28882.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28882.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59436.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60383.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59436.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40577.log.gz 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59436.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr: 92.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48304.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50100.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52882.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60383.log: /var/log/ceph/cluster1-client.mirror.48304.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60383.log.gz 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45501.log 2026-02-18T07:11:57.935 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.48304.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33043.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52882.log: /var/log/ceph/cluster2-client.mirror.45501.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45501.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52882.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56195.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68402.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33043.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54231.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.33043.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56195.log: /var/log/ceph/cluster2-client.admin.68402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53692.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56195.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54231.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68402.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54231.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37208.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53692.log: gzip -5 --verbose 87.7% -- /var/log/ceph/cluster1-client.mirror.0.32221.log 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53692.log.gz 2026-02-18T07:11:57.936 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58558.log 2026-02-18T07:11:57.937 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37208.log.gz/var/log/ceph/cluster1-client.mirror.0.32221.log: 2026-02-18T07:11:57.940 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.32221.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32648.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40834.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58558.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67908.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58558.log.gz/var/log/ceph/cluster2-client.mirror.32648.log: 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40834.log: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52101.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.32648.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40834.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.67908.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.67908.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53889.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39982.log 87.6% 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.52101.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32923.log 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53889.log.gz 2026-02-18T07:11:57.941 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39982.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40972.log.gz 2026-02-18T07:11:57.946 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56133.log 2026-02-18T07:11:57.946 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32923.log: gzip 87.9% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35395.log 2026-02-18T07:11:57.946 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.32923.log.gz 2026-02-18T07:11:57.946 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67690.log 2026-02-18T07:11:57.946 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47493.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35395.log: /var/log/ceph/cluster1-client.mirror.67690.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67690.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.35395.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64769.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47493.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56133.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49312.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47493.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.64769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50919.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.49312.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60991.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.64769.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.50919.log: -- replaced with /var/log/ceph/cluster1-client.mirror.49312.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40105.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50919.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37521.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60991.log: /var/log/ceph/cluster2-client.mirror.40105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51030.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60991.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40281.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51030.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51030.log.gz 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37541.log 2026-02-18T07:11:57.947 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40281.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40281.log.gz 2026-02-18T07:11:57.949 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50433.log 2026-02-18T07:11:57.949 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37541.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37521.log.gz 2026-02-18T07:11:57.950 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46549.log 2026-02-18T07:11:57.950 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50433.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50433.log.gz 2026-02-18T07:11:57.950 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42143.log 2026-02-18T07:11:57.950 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46549.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.46549.log.gz 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47879.log 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42143.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42143.log.gz 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37541.log.gz 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36803.log 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50592.log 2026-02-18T07:11:57.951 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47879.log: /var/log/ceph/cluster2-client.mirror.36803.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36803.log.gz 2026-02-18T07:11:57.952 INFO:teuthology.orchestra.run.trial051.stderr: 94.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53779.log 2026-02-18T07:11:57.952 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40105.log.gz 2026-02-18T07:11:57.952 INFO:teuthology.orchestra.run.trial051.stderr: 92.2%/var/log/ceph/cluster2-client.mirror.50592.log: -- replaced with /var/log/ceph/cluster1-client.mirror.47879.log.gz 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34045.log 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50592.log.gz 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28881.log 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34045.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53779.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56595.log 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.34045.log.gz 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28881.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28881.log.gz 2026-02-18T07:11:57.954 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45662.log 2026-02-18T07:11:57.955 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63781.log 2026-02-18T07:11:57.955 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45662.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67580.log 2026-02-18T07:11:57.955 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56595.log.gz 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42165.log 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67580.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63781.log.gz 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45662.log.gz 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67580.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35022.log 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57090.log 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42165.log: /var/log/ceph/cluster1-client.mirror.35022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29857.log 2026-02-18T07:11:57.956 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35022.log.gz 2026-02-18T07:11:57.957 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57090.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36108.log 2026-02-18T07:11:57.957 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29857.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57090.log.gz 2026-02-18T07:11:57.957 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29857.log.gz 2026-02-18T07:11:57.957 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67790.log 2026-02-18T07:11:57.958 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32465.log 2026-02-18T07:11:57.958 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36108.log: /var/log/ceph/cluster2-client.mirror.67790.log: 93.8% 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67790.log.gz 2026-02-18T07:11:57.958 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40893.log.gz 89.0% 2026-02-18T07:11:57.958 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.36108.log.gz 2026-02-18T07:11:57.959 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64872.log 2026-02-18T07:11:57.959 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49481.log 2026-02-18T07:11:57.959 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32465.log.gz 2026-02-18T07:11:57.959 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64872.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67060.log 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64872.log.gz 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33686.log 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49481.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49481.log.gz 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51549.log 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67060.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.43884.log 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.67060.log.gz 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33686.log: /var/log/ceph/cluster2-client.mirror.51549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47941.log 2026-02-18T07:11:57.960 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.43884.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51549.log.gz 2026-02-18T07:11:57.961 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54888.log 2026-02-18T07:11:57.961 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47941.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.47941.log.gz 2026-02-18T07:11:57.962 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61126.log 2026-02-18T07:11:57.962 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54888.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54888.log.gz 2026-02-18T07:11:57.963 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34223.log 2026-02-18T07:11:57.963 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61126.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61126.log.gz 2026-02-18T07:11:57.964 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34181.log 2026-02-18T07:11:57.964 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34223.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34223.log.gz 2026-02-18T07:11:57.964 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.33686.log.gz 2026-02-18T07:11:57.964 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53202.log 2026-02-18T07:11:57.965 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50341.log 2026-02-18T07:11:57.965 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34181.log: /var/log/ceph/cluster1-client.mirror.53202.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53202.log.gz 2026-02-18T07:11:57.968 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34181.log.gz 2026-02-18T07:11:57.969 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51671.log 2026-02-18T07:11:57.969 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66706.log 2026-02-18T07:11:57.969 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51671.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51671.log.gz 2026-02-18T07:11:57.970 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57296.log 2026-02-18T07:11:57.971 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50341.log.gz 2026-02-18T07:11:57.971 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54166.log 2026-02-18T07:11:57.971 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57296.log: 93.8% 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66706.log.gz 2026-02-18T07:11:57.971 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.38994.log.gz 2026-02-18T07:11:57.971 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61306.log 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.65000.log 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57296.log.gz 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54166.log: /var/log/ceph/cluster1-client.mirror.61306.log: 87.5%gzip -5 --verbose -- 86.6% /var/log/ceph/cluster1-client.mirror.44561.log 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.61306.log.gz 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.54166.log.gz 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.65000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50059.log 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34021.log 2026-02-18T07:11:57.973 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44561.log.gz 2026-02-18T07:11:57.974 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48925.log 2026-02-18T07:11:57.974 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34021.log: 87.8% 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50059.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.0.65000.log.gz 2026-02-18T07:11:57.974 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38935.log 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.34021.log.gz 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54094.log 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48925.log: /var/log/ceph/cluster2-client.mirror.38935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66542.log 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38935.log.gz 2026-02-18T07:11:57.975 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54094.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48925.log.gz 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54094.log.gz 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29514.log 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65709.log 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66542.log: /var/log/ceph/cluster1-client.admin.29514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47859.log 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29514.log.gz 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65709.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66542.log.gz 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65709.log.gz 2026-02-18T07:11:57.976 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46642.log 2026-02-18T07:11:57.977 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46965.log 2026-02-18T07:11:57.977 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47859.log: /var/log/ceph/cluster2-client.mirror.46642.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57338.log 2026-02-18T07:11:57.977 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46965.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46965.log.gz 2026-02-18T07:11:57.977 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 92.5% --verbose -- /var/log/ceph/cluster2-client.mirror.59167.log 2026-02-18T07:11:57.978 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46642.log.gz 2026-02-18T07:11:57.978 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47859.log.gz 2026-02-18T07:11:57.978 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57338.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57338.log.gz 2026-02-18T07:11:57.978 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29759.log 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48063.log 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29759.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59167.log.gz 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29759.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47565.log 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50180.log 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48063.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.48063.log.gz 2026-02-18T07:11:57.979 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47565.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47565.log.gz 2026-02-18T07:11:57.980 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53313.log 2026-02-18T07:11:57.980 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46818.log 2026-02-18T07:11:57.980 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53313.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50180.log.gz 2026-02-18T07:11:57.980 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52734.log 2026-02-18T07:11:57.981 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53313.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38505.log 2026-02-18T07:11:57.981 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.981 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46818.log: /var/log/ceph/cluster1-client.mirror.52734.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46818.log.gz 2026-02-18T07:11:57.981 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52734.log.gz 2026-02-18T07:11:57.981 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57152.log 2026-02-18T07:11:57.982 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38505.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56619.log 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57152.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38505.log.gz 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66054.log 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57152.log.gz 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41152.log 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56619.log: /var/log/ceph/cluster2-client.mirror.66054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59787.log 2026-02-18T07:11:57.983 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56619.log.gz 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41152.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66054.log.gz 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41152.log.gz 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60428.log 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.59787.log: /var/log/ceph/cluster1-client.mirror.54912.log 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58765.log 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59787.log.gz 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60428.log: /var/log/ceph/cluster1-client.mirror.54912.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60428.log.gz 2026-02-18T07:11:57.984 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54912.log.gz 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48369.log 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65109.log 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58765.log: /var/log/ceph/cluster1-client.mirror.48369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65401.log 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65109.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58765.log.gz 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40340.log 2026-02-18T07:11:57.985 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65109.log.gz 2026-02-18T07:11:57.986 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65401.log: 92.2% 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48369.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.65401.log.gz 2026-02-18T07:11:57.986 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:57.986 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65665.log 2026-02-18T07:11:57.987 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30519.log 2026-02-18T07:11:57.987 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40340.log: /var/log/ceph/cluster2-client.mirror.65665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39404.log 2026-02-18T07:11:57.987 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65665.log.gz/var/log/ceph/cluster1-client.admin.30519.log: 2026-02-18T07:11:57.987 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster1-client.admin.30519.log.gz 2026-02-18T07:11:57.987 INFO:teuthology.orchestra.run.trial051.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.43884.log.gz 2026-02-18T07:11:57.988 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35439.log 2026-02-18T07:11:57.988 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39404.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39404.log.gz 2026-02-18T07:11:57.989 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39161.log 2026-02-18T07:11:57.989 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35439.log: gzip -5 --verbose 84.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35439.log.gz 2026-02-18T07:11:57.989 INFO:teuthology.orchestra.run.trial051.stderr: -- /var/log/ceph/cluster1-client.mirror.34201.log 2026-02-18T07:11:57.989 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59870.log 2026-02-18T07:11:57.990 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39161.log.gz 2026-02-18T07:11:57.990 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39515.log 2026-02-18T07:11:57.990 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59870.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59870.log.gz 2026-02-18T07:11:57.991 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59704.log 2026-02-18T07:11:57.991 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.39515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58289.log 2026-02-18T07:11:57.992 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59704.log: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.39515.log.gz 2026-02-18T07:11:57.992 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34201.log.gz 2026-02-18T07:11:57.992 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59704.log.gz 2026-02-18T07:11:57.992 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30587.log 2026-02-18T07:11:57.993 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41904.log 2026-02-18T07:11:57.993 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58289.log: /var/log/ceph/cluster1-client.mirror.30587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28534.log 2026-02-18T07:11:57.993 INFO:teuthology.orchestra.run.trial051.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30587.log.gz 2026-02-18T07:11:57.994 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41904.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58289.log.gz 2026-02-18T07:11:57.996 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41132.log.gz 2026-02-18T07:11:58.000 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34924.log 2026-02-18T07:11:58.001 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28534.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28534.log.gz 2026-02-18T07:11:58.001 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49529.log 2026-02-18T07:11:58.001 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54953.log 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49529.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34924.log.gz 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58517.log 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49529.log.gz 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62520.log 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54953.log: 93.8%/var/log/ceph/cluster2-client.mirror.58517.log: -- replaced with /var/log/ceph/cluster2-client.mirror.43052.log.gz 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63304.log 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58517.log.gz 2026-02-18T07:11:58.002 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62520.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65483.log 2026-02-18T07:11:58.003 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62520.log.gz 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38585.log 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65483.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63304.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64811.log 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54953.log.gz 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr: 87.0%/var/log/ceph/cluster2-client.mirror.38585.log: -- replaced with /var/log/ceph/cluster2-client.mirror.65483.log.gz 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40755.log 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64811.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38585.log.gz 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46140.log 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64811.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38685.log 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40755.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40755.log.gz 2026-02-18T07:11:58.004 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36982.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46140.log: /var/log/ceph/cluster2-client.mirror.38685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61081.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38685.log.gz 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44633.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36982.log: /var/log/ceph/cluster1-client.mirror.61081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49019.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36982.log.gz 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.44633.log: -- replaced with /var/log/ceph/cluster1-client.mirror.61081.log.gz 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46140.log.gz 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37248.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.44633.log.gz 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65525.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49019.log: /var/log/ceph/cluster2-client.mirror.37248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67884.log 2026-02-18T07:11:58.006 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49019.log.gz 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29002.log 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37248.log.gz 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.67884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44316.log 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65525.log.gz 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.67884.log.gz 2026-02-18T07:11:58.007 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64130.log 2026-02-18T07:11:58.009 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29002.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29002.log.gz/var/log/ceph/cluster1-client.mirror.44316.log: 2026-02-18T07:11:58.009 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43819.log 2026-02-18T07:11:58.009 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44316.log.gz 2026-02-18T07:11:58.009 INFO:teuthology.orchestra.run.trial051.stderr:gzip/var/log/ceph/cluster2-client.mirror.64130.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59017.log 2026-02-18T07:11:58.009 INFO:teuthology.orchestra.run.trial051.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64130.log.gz 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.43819.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43819.log.gz 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64623.log 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67642.log 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59017.log.gz 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64623.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31942.log 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64623.log.gz 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65316.log 2026-02-18T07:11:58.010 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67642.log.gz 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31942.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38170.log 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65316.log: 86.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60293.log 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.65316.log.gz 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38170.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63243.log 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48182.log 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60293.log.gz 2026-02-18T07:11:58.011 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37370.log 2026-02-18T07:11:58.012 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48182.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63243.log.gz 2026-02-18T07:11:58.012 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66436.log 2026-02-18T07:11:58.012 INFO:teuthology.orchestra.run.trial051.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48182.log.gz 2026-02-18T07:11:58.012 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37370.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.37370.log.gz 2026-02-18T07:11:58.013 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57255.log 2026-02-18T07:11:58.013 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41568.log 2026-02-18T07:11:58.013 INFO:teuthology.orchestra.run.trial051.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.57255.log: -- replaced with /var/log/ceph/cluster2-client.mirror.66436.log.gz 2026-02-18T07:11:58.014 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57255.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56293.log 2026-02-18T07:11:58.014 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.014 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38170.log.gz 2026-02-18T07:11:58.014 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41568.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41568.log.gz 2026-02-18T07:11:58.015 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32537.log 2026-02-18T07:11:58.015 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64790.log 2026-02-18T07:11:58.015 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34245.log.gz 2026-02-18T07:11:58.015 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57896.log 2026-02-18T07:11:58.016 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32537.log.gz 2026-02-18T07:11:58.016 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37298.log 2026-02-18T07:11:58.016 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64790.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64790.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65993.log 2026-02-18T07:11:58.016 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.016 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57896.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39824.log.gz 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37298.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57896.log.gz 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37298.log.gz 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64109.log 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60316.log 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56293.log.gz 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.65993.log.gz 2026-02-18T07:11:58.017 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61801.log 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32835.log 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60316.log.gz 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61801.log: 86.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51008.log 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.61801.log.gz 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64109.log.gz 2026-02-18T07:11:58.018 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.32835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64586.log 2026-02-18T07:11:58.019 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51008.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.32835.log.gz 2026-02-18T07:11:58.019 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57028.log 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64586.log: 79.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62228.log 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64586.log.gz 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53271.log 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57028.log.gz 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr: 92.2%/var/log/ceph/cluster1-client.mirror.62228.log: -- replaced with /var/log/ceph/cluster1-client.mirror.51008.log.gz 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr:gzip 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62228.log.gz 2026-02-18T07:11:58.020 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44222.log 2026-02-18T07:11:58.021 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49381.log 2026-02-18T07:11:58.021 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41983.log 2026-02-18T07:11:58.021 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44222.log.gz 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57792.log 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53271.log.gz 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38107.log 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57792.log: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63465.log 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.49381.log.gz 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38107.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53070.log 2026-02-18T07:11:58.022 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63407.log 2026-02-18T07:11:58.023 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63465.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57792.log.gz 2026-02-18T07:11:58.023 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53070.log: gzip -5 --verbose 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63465.log.gz 2026-02-18T07:11:58.023 INFO:teuthology.orchestra.run.trial051.stderr: -- /var/log/ceph/cluster2-client.mirror.64442.log 2026-02-18T07:11:58.023 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37478.log 2026-02-18T07:11:58.023 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63407.log.gz 2026-02-18T07:11:58.024 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33751.log 2026-02-18T07:11:58.024 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64442.log.gz 2026-02-18T07:11:58.024 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37478.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53070.log.gz 2026-02-18T07:11:58.024 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48903.log 2026-02-18T07:11:58.025 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61778.log 2026-02-18T07:11:58.025 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33751.log: /var/log/ceph/cluster1-client.mirror.48903.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38107.log.gz 2026-02-18T07:11:58.025 INFO:teuthology.orchestra.run.trial051.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33751.log.gz 2026-02-18T07:11:58.026 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40211.log 2026-02-18T07:11:58.026 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61778.log: 92.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58599.log 2026-02-18T07:11:58.026 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.48903.log.gz 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61778.log.gz 2026-02-18T07:11:58.026 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.026 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35134.log 2026-02-18T07:11:58.027 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51475.log 2026-02-18T07:11:58.027 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35134.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40211.log.gz 2026-02-18T07:11:58.027 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31133.log 2026-02-18T07:11:58.027 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37478.log.gz 2026-02-18T07:11:58.028 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58599.log.gz 2026-02-18T07:11:58.028 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52173.log 2026-02-18T07:11:58.028 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31133.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31133.log.gz 2026-02-18T07:11:58.028 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55563.log 2026-02-18T07:11:58.028 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38625.log 2026-02-18T07:11:58.029 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55563.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.52173.log.gz 2026-02-18T07:11:58.029 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51475.log.gz 2026-02-18T07:11:58.029 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55563.log.gz 2026-02-18T07:11:58.029 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62817.log 2026-02-18T07:11:58.030 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59981.log 2026-02-18T07:11:58.030 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62817.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38625.log.gz 2026-02-18T07:11:58.030 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49043.log 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62817.log.gz 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35134.log.gz 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.59981.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59746.log 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49043.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59981.log.gz 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67815.log 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49043.log.gz 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53375.log 2026-02-18T07:11:58.031 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.59746.log.gz 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67815.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67815.log.gz 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66604.log 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53375.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63448.log 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.66604.log: -- replaced with /var/log/ceph/cluster2-client.mirror.53375.log.gz 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65274.log 2026-02-18T07:11:58.032 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66604.log.gz 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55334.log 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63448.log.gz 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.65274.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37584.log 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56924.log 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65274.log.gz 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37584.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55334.log.gz 2026-02-18T07:11:58.033 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42691.log 2026-02-18T07:11:58.034 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63616.log 2026-02-18T07:11:58.034 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56924.log: /var/log/ceph/cluster2-client.mirror.42691.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60361.log 2026-02-18T07:11:58.034 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42691.log.gz 2026-02-18T07:11:58.034 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63616.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63616.log.gz 2026-02-18T07:11:58.035 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56924.log.gz 2026-02-18T07:11:58.035 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32561.log 2026-02-18T07:11:58.036 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60361.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60361.log.gz 2026-02-18T07:11:58.036 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36332.log 2026-02-18T07:11:58.036 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62003.log 2026-02-18T07:11:58.036 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.32561.log.gz 2026-02-18T07:11:58.036 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42624.log 2026-02-18T07:11:58.037 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37584.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31167.log 2026-02-18T07:11:58.037 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.037 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62003.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62003.log.gz 2026-02-18T07:11:58.037 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35818.log.gz 2026-02-18T07:11:58.037 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42624.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42624.log.gz 2026-02-18T07:11:58.038 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67101.log 2026-02-18T07:11:58.038 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31167.log: 64.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31167.log.gz 2026-02-18T07:11:58.038 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53005.log 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37412.log 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67101.log: /var/log/ceph/cluster1-client.mirror.53005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55714.log 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67101.log.gz 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53005.log.gz 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43295.log 2026-02-18T07:11:58.039 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36616.log 2026-02-18T07:11:58.040 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43531.log 2026-02-18T07:11:58.040 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55714.log.gz 2026-02-18T07:11:58.040 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58207.log 2026-02-18T07:11:58.042 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43531.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37412.log.gz 2026-02-18T07:11:58.043 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36616.log.gz 2026-02-18T07:11:58.043 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52280.log 2026-02-18T07:11:58.043 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64955.log 2026-02-18T07:11:58.044 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58207.log: /var/log/ceph/cluster2-client.mirror.52280.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58207.log.gz 2026-02-18T07:11:58.044 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62048.log 2026-02-18T07:11:58.044 INFO:teuthology.orchestra.run.trial051.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52280.log.gz 2026-02-18T07:11:58.045 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64955.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64955.log.gz 2026-02-18T07:11:58.045 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42192.log 2026-02-18T07:11:58.045 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55150.log 2026-02-18T07:11:58.045 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62048.log: /var/log/ceph/cluster2-client.mirror.42192.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62048.log.gz 2026-02-18T07:11:58.046 INFO:teuthology.orchestra.run.trial051.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42192.log.gz 2026-02-18T07:11:58.046 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59766.log 2026-02-18T07:11:58.046 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45573.log 2026-02-18T07:11:58.047 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55150.log: /var/log/ceph/cluster2-client.mirror.59766.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55150.log.gz 2026-02-18T07:11:58.047 INFO:teuthology.orchestra.run.trial051.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59766.log.gz 2026-02-18T07:11:58.047 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34638.log 2026-02-18T07:11:58.048 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46071.log 2026-02-18T07:11:58.048 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45573.log: /var/log/ceph/cluster1-client.mirror.34638.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45573.log.gz 2026-02-18T07:11:58.048 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34638.log.gz 2026-02-18T07:11:58.048 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39940.log 2026-02-18T07:11:58.049 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41052.log 2026-02-18T07:11:58.049 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46071.log: /var/log/ceph/cluster2-client.mirror.39940.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46071.log.gz 2026-02-18T07:11:58.049 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43489.log 2026-02-18T07:11:58.049 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39940.log.gz 2026-02-18T07:11:58.070 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43997.log 2026-02-18T07:11:58.070 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43489.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43489.log.gz 2026-02-18T07:11:58.071 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43295.log.gz 2026-02-18T07:11:58.072 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34834.log 2026-02-18T07:11:58.072 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43997.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43997.log.gz 2026-02-18T07:11:58.073 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51381.log 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52908.log 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51381.log: /var/log/ceph/cluster1-client.admin.34834.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51381.log.gz 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.34834.log.gz 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54395.log 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37078.log 2026-02-18T07:11:58.074 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52908.log: /var/log/ceph/cluster1-client.mirror.54395.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52908.log.gz 2026-02-18T07:11:58.075 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54395.log.gz 2026-02-18T07:11:58.075 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36204.log 2026-02-18T07:11:58.075 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55236.log 2026-02-18T07:11:58.075 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37078.log: /var/log/ceph/cluster1-client.mirror.36204.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37078.log.gz 2026-02-18T07:11:58.076 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36204.log.gz 2026-02-18T07:11:58.076 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40002.log 2026-02-18T07:11:58.076 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55236.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55236.log.gz 2026-02-18T07:11:58.077 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63678.log 2026-02-18T07:11:58.077 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57048.log 2026-02-18T07:11:58.077 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40002.log: /var/log/ceph/cluster2-client.mirror.63678.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40002.log.gz 2026-02-18T07:11:58.078 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47377.log 2026-02-18T07:11:58.078 INFO:teuthology.orchestra.run.trial051.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63678.log.gz 2026-02-18T07:11:58.079 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57048.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57048.log.gz 2026-02-18T07:11:58.079 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58806.log 2026-02-18T07:11:58.079 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40735.log 2026-02-18T07:11:58.080 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47377.log: /var/log/ceph/cluster2-client.mirror.58806.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58806.log.gz 2026-02-18T07:11:58.080 INFO:teuthology.orchestra.run.trial051.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.47377.log.gz 2026-02-18T07:11:58.081 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37346.log 2026-02-18T07:11:58.081 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39441.log 2026-02-18T07:11:58.082 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40735.log: /var/log/ceph/cluster1-client.mirror.37346.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37346.log.gz 2026-02-18T07:11:58.087 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65952.log 2026-02-18T07:11:58.087 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39441.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39441.log.gz 2026-02-18T07:11:58.087 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35711.log 2026-02-18T07:11:58.088 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65952.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65952.log.gz 2026-02-18T07:11:58.088 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61666.log 2026-02-18T07:11:58.092 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35711.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35711.log.gz 2026-02-18T07:11:58.092 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58227.log 2026-02-18T07:11:58.093 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61666.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61666.log.gz 2026-02-18T07:11:58.093 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43585.log 2026-02-18T07:11:58.094 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58227.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58227.log.gz 2026-02-18T07:11:58.094 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42817.log 2026-02-18T07:11:58.095 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43585.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43585.log.gz 2026-02-18T07:11:58.095 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55996.log 2026-02-18T07:11:58.096 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42817.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42817.log.gz 2026-02-18T07:11:58.096 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58020.log 2026-02-18T07:11:58.097 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55996.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55996.log.gz 2026-02-18T07:11:58.097 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47901.log 2026-02-18T07:11:58.098 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58020.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58020.log.gz 2026-02-18T07:11:58.098 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60833.log 2026-02-18T07:11:58.098 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47901.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47901.log.gz 2026-02-18T07:11:58.099 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64167.log 2026-02-18T07:11:58.099 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60833.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60833.log.gz 2026-02-18T07:11:58.100 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44826.log 2026-02-18T07:11:58.100 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64167.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64167.log.gz 2026-02-18T07:11:58.100 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45123.log 2026-02-18T07:11:58.101 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44826.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44826.log.gz 2026-02-18T07:11:58.101 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44388.log 2026-02-18T07:11:58.101 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45123.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45123.log.gz 2026-02-18T07:11:58.102 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.39315.log 2026-02-18T07:11:58.102 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44388.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44388.log.gz 2026-02-18T07:11:58.102 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30706.log 2026-02-18T07:11:58.104 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.39315.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62369.log.gz 2026-02-18T07:11:58.106 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34768.log 2026-02-18T07:11:58.106 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30706.log: 54.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30706.log.gz 2026-02-18T07:11:58.107 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68451.log 2026-02-18T07:11:58.107 INFO:teuthology.orchestra.run.trial051.stderr: 93.8%/var/log/ceph/cluster2-client.mirror.34768.log: -- replaced with /var/log/ceph/cluster2-client.mirror.42859.log.gz 2026-02-18T07:11:58.107 INFO:teuthology.orchestra.run.trial051.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34768.log.gz 2026-02-18T07:11:58.108 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66370.log 2026-02-18T07:11:58.108 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5/var/log/ceph/cluster2-client.admin.68451.log: --verbose -- /var/log/ceph/cluster2-client.admin.29318.log 2026-02-18T07:11:58.108 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68451.log.gz 2026-02-18T07:11:58.113 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66370.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38915.log.gz 2026-02-18T07:11:58.113 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62390.log 2026-02-18T07:11:58.113 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29318.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54211.log 2026-02-18T07:11:58.113 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29318.log.gz 2026-02-18T07:11:58.114 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44850.log 2026-02-18T07:11:58.114 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62390.log.gz/var/log/ceph/cluster2-client.mirror.54211.log: 2026-02-18T07:11:58.114 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31201.log 2026-02-18T07:11:58.115 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44850.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44850.log.gz 2026-02-18T07:11:58.115 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54211.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48427.log 2026-02-18T07:11:58.115 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.116 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55382.log 2026-02-18T07:11:58.116 INFO:teuthology.orchestra.run.trial051.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31201.log.gz 2026-02-18T07:11:58.116 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41290.log 2026-02-18T07:11:58.117 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55382.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48427.log.gz 2026-02-18T07:11:58.117 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55382.log.gz 2026-02-18T07:11:58.117 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37607.log 2026-02-18T07:11:58.117 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62251.log 2026-02-18T07:11:58.120 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37607.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37607.log.gz 2026-02-18T07:11:58.120 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52782.log 2026-02-18T07:11:58.121 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62251.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62251.log.gz 2026-02-18T07:11:58.121 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42251.log 2026-02-18T07:11:58.122 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52782.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52782.log.gz 2026-02-18T07:11:58.122 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40061.log 2026-02-18T07:11:58.125 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42251.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42067.log.gz 2026-02-18T07:11:58.125 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65972.log 2026-02-18T07:11:58.135 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40061.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.39315.log.gz 2026-02-18T07:11:58.136 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.32334.log 2026-02-18T07:11:58.136 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65972.log: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65972.log.gz 2026-02-18T07:11:58.136 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49642.log 2026-02-18T07:11:58.145 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.1.32334.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67330.log 2026-02-18T07:11:58.160 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49642.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49642.log.gz 2026-02-18T07:11:58.163 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58123.log 2026-02-18T07:11:58.163 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.67330.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67330.log.gz 2026-02-18T07:11:58.172 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47190.log 2026-02-18T07:11:58.184 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58123.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58123.log.gz 2026-02-18T07:11:58.189 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52236.log 2026-02-18T07:11:58.190 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47190.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47190.log.gz 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.32334.log.gz 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49553.log 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52236.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52236.log.gz 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42523.log 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49553.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49553.log.gz 2026-02-18T07:11:58.194 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40792.log 2026-02-18T07:11:58.196 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42523.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42523.log.gz 2026-02-18T07:11:58.196 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37166.log 2026-02-18T07:11:58.196 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40792.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40792.log.gz 2026-02-18T07:11:58.197 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37392.log 2026-02-18T07:11:58.199 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37166.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37166.log.gz 2026-02-18T07:11:58.200 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33773.log 2026-02-18T07:11:58.203 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37392.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37392.log.gz 2026-02-18T07:11:58.203 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52803.log 2026-02-18T07:11:58.204 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33773.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40498.log.gz 2026-02-18T07:11:58.205 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66116.log 2026-02-18T07:11:58.206 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52803.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52803.log.gz 2026-02-18T07:11:58.206 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34790.log.gz 2026-02-18T07:11:58.207 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41189.log 2026-02-18T07:11:58.214 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66116.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43030.log 2026-02-18T07:11:58.214 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.66116.log.gz 2026-02-18T07:11:58.214 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25617.log 2026-02-18T07:11:58.214 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41189.log: /var/log/ceph/cluster2-client.mirror.43030.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41189.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43030.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43176.log 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.25617.log: -- /var/log/ceph/cluster1-client.mirror.62093.log 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.25617.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64679.log 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62093.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43176.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62093.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60023.log 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.64679.log: -- /var/log/ceph/cluster1-client.mirror.37799.log 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.60023.log: -- replaced with /var/log/ceph/cluster1-client.mirror.64679.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60023.log.gz 2026-02-18T07:11:58.215 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39024.log 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37799.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63593.log 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39024.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39024.log.gz 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39271.log 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63593.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.63593.log.gz 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53399.log 2026-02-18T07:11:58.216 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39271.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37799.log.gz 2026-02-18T07:11:58.224 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67351.log 2026-02-18T07:11:58.224 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53399.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53399.log.gz 2026-02-18T07:11:58.224 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63842.log 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67351.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67351.log.gz 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30471.log 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32670.log.gz 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49156.log 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.30471.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63842.log.gz 2026-02-18T07:11:58.233 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30471.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50791.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62026.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49156.log: /var/log/ceph/cluster1-client.mirror.50791.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49156.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50791.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52027.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62026.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62026.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37322.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.30231.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37322.log: /var/log/ceph/cluster2-client.mirror.52027.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37322.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52027.log.gz 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47015.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.3.30231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44087.log 2026-02-18T07:11:58.234 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47015.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47015.log.gz 2026-02-18T07:11:58.235 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53250.log 2026-02-18T07:11:58.235 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44087.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.44087.log.gz 2026-02-18T07:11:58.235 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52125.log 2026-02-18T07:11:58.235 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53250.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53250.log.gz 2026-02-18T07:11:58.236 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.63887.log 2026-02-18T07:11:58.236 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52125.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52125.log.gz 2026-02-18T07:11:58.237 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47443.log 2026-02-18T07:11:58.238 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.63887.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.63887.log.gz 2026-02-18T07:11:58.238 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46525.log 2026-02-18T07:11:58.238 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.47443.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.47443.log.gz 2026-02-18T07:11:58.239 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56882.log 2026-02-18T07:11:58.239 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46525.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.46525.log.gz 2026-02-18T07:11:58.240 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38193.log 2026-02-18T07:11:58.240 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56882.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56882.log.gz 2026-02-18T07:11:58.241 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38766.log 2026-02-18T07:11:58.244 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38193.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38193.log.gz 2026-02-18T07:11:58.245 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52678.log 2026-02-18T07:11:58.245 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38766.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38766.log.gz 2026-02-18T07:11:58.247 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62797.log 2026-02-18T07:11:58.247 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52678.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52678.log.gz 2026-02-18T07:11:58.247 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30231.log 2026-02-18T07:11:58.248 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62797.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62797.log.gz 2026-02-18T07:11:58.248 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49963.log 2026-02-18T07:11:58.251 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30231.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30231.log.gz 2026-02-18T07:11:58.252 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46023.log 2026-02-18T07:11:58.252 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49963.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49963.log.gz 2026-02-18T07:11:58.253 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67765.log 2026-02-18T07:11:58.254 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46023.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.46023.log.gz 2026-02-18T07:11:58.254 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60158.log 2026-02-18T07:11:58.254 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67765.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67765.log.gz 2026-02-18T07:11:58.255 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67307.log 2026-02-18T07:11:58.255 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60158.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60158.log.gz 2026-02-18T07:11:58.256 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47288.log 2026-02-18T07:11:58.256 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.67307.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67307.log.gz 2026-02-18T07:11:58.257 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68598.log 2026-02-18T07:11:58.257 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47288.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47288.log.gz 2026-02-18T07:11:58.257 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35198.log 2026-02-18T07:11:58.258 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.68598.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68598.log.gz 2026-02-18T07:11:58.258 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61936.log 2026-02-18T07:11:58.267 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42646.log 2026-02-18T07:11:58.268 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61936.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61936.log.gz 2026-02-18T07:11:58.281 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34590.log 2026-02-18T07:11:58.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42646.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61396.log 2026-02-18T07:11:58.288 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34590.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.34590.log.gz 2026-02-18T07:11:58.292 INFO:teuthology.orchestra.run.trial051.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.3.30231.log.gz 2026-02-18T07:11:58.293 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60113.log 2026-02-18T07:11:58.293 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61396.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61396.log.gz 2026-02-18T07:11:58.302 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51136.log 2026-02-18T07:11:58.302 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60113.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60113.log.gz 2026-02-18T07:11:58.303 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39463.log.gz 2026-02-18T07:11:58.305 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42433.log 2026-02-18T07:11:58.305 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51136.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51136.log.gz 2026-02-18T07:11:58.305 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48838.log 2026-02-18T07:11:58.315 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45413.log 2026-02-18T07:11:58.316 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48838.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48838.log.gz 2026-02-18T07:11:58.333 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39923.log 2026-02-18T07:11:58.334 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45413.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45413.log.gz 2026-02-18T07:11:58.336 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39982.log.gz 2026-02-18T07:11:58.338 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52578.log 2026-02-18T07:11:58.339 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39923.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.39923.log.gz 2026-02-18T07:11:58.339 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60698.log 2026-02-18T07:11:58.339 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52578.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52578.log.gz 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38445.log 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60698.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60698.log.gz 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43798.log 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38445.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38445.log.gz 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62161.log 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43798.log: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.43798.log.gz 2026-02-18T07:11:58.344 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63822.log 2026-02-18T07:11:58.345 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62161.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62161.log.gz 2026-02-18T07:11:58.345 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49067.log 2026-02-18T07:11:58.345 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63822.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63822.log.gz 2026-02-18T07:11:58.345 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51828.log 2026-02-18T07:11:58.348 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49067.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49067.log.gz 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62622.log 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51828.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51828.log.gz 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33316.log 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62622.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62622.log.gz 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57110.log 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33316.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.33316.log.gz 2026-02-18T07:11:58.349 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65753.log 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57110.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57110.log.gz 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41428.log 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65753.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65753.log.gz 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28734.log 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41428.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41428.log.gz 2026-02-18T07:11:58.354 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55674.log 2026-02-18T07:11:58.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28734.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28734.log.gz 2026-02-18T07:11:58.355 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47742.log 2026-02-18T07:11:58.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55674.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55674.log.gz 2026-02-18T07:11:58.355 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46119.log 2026-02-18T07:11:58.355 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47742.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47742.log.gz 2026-02-18T07:11:58.359 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56245.log 2026-02-18T07:11:58.359 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46119.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46119.log.gz 2026-02-18T07:11:58.359 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60653.log 2026-02-18T07:11:58.359 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56245.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56245.log.gz 2026-02-18T07:11:58.359 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32124.log 2026-02-18T07:11:58.360 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60653.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60653.log.gz 2026-02-18T07:11:58.360 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37713.log 2026-02-18T07:11:58.360 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.32124.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.32124.log.gz 2026-02-18T07:11:58.360 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64483.log 2026-02-18T07:11:58.362 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37713.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37713.log.gz 2026-02-18T07:11:58.363 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49577.log 2026-02-18T07:11:58.364 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64483.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64483.log.gz 2026-02-18T07:11:58.364 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36288.log 2026-02-18T07:11:58.364 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49577.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49577.log.gz 2026-02-18T07:11:58.365 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58496.log 2026-02-18T07:11:58.369 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36288.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36288.log.gz 2026-02-18T07:11:58.369 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59622.log 2026-02-18T07:11:58.369 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58496.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58496.log.gz 2026-02-18T07:11:58.374 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41551.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59622.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59622.log.gz 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66311.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41551.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41551.log.gz 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29001.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66311.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66311.log.gz 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47814.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29001.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29001.log.gz 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41110.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47814.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47814.log.gz 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48256.log 2026-02-18T07:11:58.375 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41110.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41110.log.gz 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47087.log 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48256.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48256.log.gz 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48525.log 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47087.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47087.log.gz 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33171.log 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48525.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48525.log.gz 2026-02-18T07:11:58.379 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29177.log 2026-02-18T07:11:58.387 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35154.log 2026-02-18T07:11:58.387 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29177.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29177.log.gz 2026-02-18T07:11:58.397 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41290.log.gz 2026-02-18T07:11:58.402 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39742.log.gz 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62643.log 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40735.log.gz 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56861.log 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46276.log 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62643.log: /var/log/ceph/cluster2-client.mirror.56861.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62643.log.gz 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50726.log 2026-02-18T07:11:58.407 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56861.log.gz 2026-02-18T07:11:58.435 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46276.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52491.log 2026-02-18T07:11:58.435 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.46276.log.gz 2026-02-18T07:11:58.435 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50726.log: 87.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37842.log 2026-02-18T07:11:58.435 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50726.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61733.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 93.5%/var/log/ceph/cluster1-client.mirror.37842.log: -- replaced with /var/log/ceph/cluster1-client.mirror.35154.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52491.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57937.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61733.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35690.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61733.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57937.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36180.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57937.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.35690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41803.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36180.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.35690.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.36180.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38808.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39124.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.37842.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38808.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41803.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38808.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58682.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46047.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39124.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39124.log.gz 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45226.log 2026-02-18T07:11:58.436 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58682.log: /var/log/ceph/cluster1-client.mirror.46047.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58372.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% 87.7%/var/log/ceph/cluster2-client.mirror.45226.log: -- replaced with /var/log/ceph/cluster1-client.mirror.46047.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58682.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.45226.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47264.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39041.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.6%/var/log/ceph/cluster1-client.mirror.47264.log: -- replaced with /var/log/ceph/cluster2-client.mirror.58372.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57978.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47264.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47063.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39041.log: /var/log/ceph/cluster2-client.mirror.57978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42209.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47063.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39041.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.47063.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38665.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57978.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41470.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42209.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56779.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38665.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42209.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41470.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41470.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38665.log.gz 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36058.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32897.log 2026-02-18T07:11:58.437 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56779.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45252.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36058.log: /var/log/ceph/cluster2-client.mirror.32897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29661.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45252.log: 89.3% 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36058.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.32897.log.gz 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45252.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44111.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29661.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29661.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45276.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54024.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44111.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45276.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61058.log 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45276.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30749.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54024.log.gz/var/log/ceph/cluster1-client.mirror.61058.log: 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61058.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59332.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28733.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.30749.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30749.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34950.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62499.log 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28733.log: /var/log/ceph/cluster2-client.mirror.34950.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28733.log.gz 2026-02-18T07:11:58.438 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34950.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 90.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54975.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59332.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61576.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62499.log: /var/log/ceph/cluster2-client.mirror.54975.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46435.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62499.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61576.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54975.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61576.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41961.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61823.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46435.log: /var/log/ceph/cluster2-client.mirror.41961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44159.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.61823.log: -- replaced with /var/log/ceph/cluster2-client.mirror.41961.log.gz 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46435.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61823.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55587.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35342.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44159.log: /var/log/ceph/cluster1-client.mirror.55587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44063.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44159.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55587.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.35342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62992.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44063.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.35342.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44063.log.gz 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42045.log 2026-02-18T07:11:58.439 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62992.log: gzip 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62992.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63658.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42045.log: gzip -5 --verbose 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42045.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: -- /var/log/ceph/cluster2-client.mirror.37736.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54571.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.37736.log: -- replaced with /var/log/ceph/cluster2-client.mirror.63658.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38365.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54545.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54571.log: /var/log/ceph/cluster2-client.mirror.38365.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54571.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43531.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38365.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53226.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.54545.log: --verbose -- /var/log/ceph/cluster1-client.mirror.36499.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56712.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 89.1%/var/log/ceph/cluster1-client.mirror.53226.log: -- replaced with /var/log/ceph/cluster2-client.mirror.54545.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster1-client.mirror.36499.log: -- replaced with /var/log/ceph/cluster1-client.mirror.53226.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40318.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56712.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.36499.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42545.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37736.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40318.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45822.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40318.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45348.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56712.log.gz 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42545.log: /var/log/ceph/cluster1-client.mirror.45822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56799.log 2026-02-18T07:11:58.440 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45822.log.gz 2026-02-18T07:11:58.441 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45348.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45348.log.gz 2026-02-18T07:11:58.444 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46597.log 2026-02-18T07:11:58.444 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56799.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.58953.log 2026-02-18T07:11:58.445 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46597.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67498.log 2026-02-18T07:11:58.445 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46597.log.gz 2026-02-18T07:11:58.445 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.0.58953.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56799.log.gz 2026-02-18T07:11:58.445 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58579.log 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52149.log 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58579.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67498.log.gz 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58579.log.gz 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41030.log 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28299.log 2026-02-18T07:11:58.446 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52149.log.gz 2026-02-18T07:11:58.447 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53913.log 2026-02-18T07:11:58.447 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28299.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41030.log.gz 2026-02-18T07:11:58.447 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28299.log.gz 2026-02-18T07:11:58.447 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61418.log 2026-02-18T07:11:58.448 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51599.log 2026-02-18T07:11:58.448 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61418.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53913.log.gz 2026-02-18T07:11:58.448 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61418.log.gz 2026-02-18T07:11:58.448 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58953.log 2026-02-18T07:11:58.448 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50550.log 2026-02-18T07:11:58.449 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58953.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51599.log.gz 2026-02-18T07:11:58.449 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58953.log.gz 2026-02-18T07:11:58.449 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65546.log 2026-02-18T07:11:58.449 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50550.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37030.log 2026-02-18T07:11:58.450 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65546.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.58953.log.gz 2026-02-18T07:11:58.450 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65546.log.gz 2026-02-18T07:11:58.450 INFO:teuthology.orchestra.run.trial051.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50550.log.gz 2026-02-18T07:11:58.450 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50654.log 2026-02-18T07:11:58.450 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54708.log 2026-02-18T07:11:58.451 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34614.log 2026-02-18T07:11:58.451 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.37030.log.gz 2026-02-18T07:11:58.451 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50654.log: /var/log/ceph/cluster1-client.mirror.54708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63507.log 2026-02-18T07:11:58.451 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50654.log.gz 2026-02-18T07:11:58.451 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34683.log 2026-02-18T07:11:58.452 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63507.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34614.log.gz 2026-02-18T07:11:58.452 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54708.log.gz 2026-02-18T07:11:58.452 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37693.log 2026-02-18T07:11:58.452 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52077.log 2026-02-18T07:11:58.456 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.34683.log: /var/log/ceph/cluster2-client.mirror.37693.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37693.log.gz 2026-02-18T07:11:58.456 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34683.log.gz 2026-02-18T07:11:58.457 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54419.log 2026-02-18T07:11:58.457 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66328.log 2026-02-18T07:11:58.457 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52077.log.gz 2026-02-18T07:11:58.457 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52702.log 2026-02-18T07:11:58.458 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66328.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54419.log.gz 2026-02-18T07:11:58.458 INFO:teuthology.orchestra.run.trial051.stderr: 88.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42025.log 2026-02-18T07:11:58.458 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.66328.log.gz 2026-02-18T07:11:58.458 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52702.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61846.log 2026-02-18T07:11:58.459 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42025.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52702.log.gz 2026-02-18T07:11:58.459 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42025.log.gz 2026-02-18T07:11:58.459 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32808.log 2026-02-18T07:11:58.459 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38565.log 2026-02-18T07:11:58.460 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61846.log: /var/log/ceph/cluster1-client.admin.32808.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61846.log.gz 2026-02-18T07:11:58.460 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.32808.log.gz 2026-02-18T07:11:58.460 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29269.log 2026-02-18T07:11:58.460 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52654.log 2026-02-18T07:11:58.461 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38565.log: /var/log/ceph/cluster2-client.admin.29269.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29269.log.gz 2026-02-18T07:11:58.461 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38565.log.gz 2026-02-18T07:11:58.462 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67372.log 2026-02-18T07:11:58.462 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37435.log 2026-02-18T07:11:58.462 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52654.log.gz 2026-02-18T07:11:58.463 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.67372.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36225.log 2026-02-18T07:11:58.463 INFO:teuthology.orchestra.run.trial051.stderr: 86.6%/var/log/ceph/cluster2-client.mirror.37435.log: -- replaced with /var/log/ceph/cluster1-client.mirror.67372.log.gz 2026-02-18T07:11:58.463 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54142.log 2026-02-18T07:11:58.466 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36225.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37435.log.gz 2026-02-18T07:11:58.467 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47630.log 2026-02-18T07:11:58.467 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54142.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36225.log.gz 2026-02-18T07:11:58.467 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54142.log.gz 2026-02-18T07:11:58.467 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62455.log 2026-02-18T07:11:58.467 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29031.log 2026-02-18T07:11:58.468 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47630.log: /var/log/ceph/cluster1-client.mirror.62455.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62455.log.gz 2026-02-18T07:11:58.469 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42960.log 2026-02-18T07:11:58.469 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29031.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29031.log.gz 2026-02-18T07:11:58.469 INFO:teuthology.orchestra.run.trial051.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.47630.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45549.log 2026-02-18T07:11:58.469 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.470 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42960.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60788.log 2026-02-18T07:11:58.470 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45549.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45549.log.gz 2026-02-18T07:11:58.470 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40993.log 2026-02-18T07:11:58.471 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60788.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60788.log.gz 2026-02-18T07:11:58.471 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67186.log 2026-02-18T07:11:58.472 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40993.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40993.log.gz 2026-02-18T07:11:58.472 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65171.log 2026-02-18T07:11:58.473 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67186.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67186.log.gz 2026-02-18T07:11:58.473 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47766.log 2026-02-18T07:11:58.473 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65171.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65171.log.gz 2026-02-18T07:11:58.474 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51802.log 2026-02-18T07:11:58.475 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47766.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47766.log.gz 2026-02-18T07:11:58.475 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67286.log 2026-02-18T07:11:58.476 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51802.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51802.log.gz 2026-02-18T07:11:58.476 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63551.log 2026-02-18T07:11:58.479 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67286.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31942.log.gz 2026-02-18T07:11:58.480 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50702.log 2026-02-18T07:11:58.480 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63551.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63551.log.gz 2026-02-18T07:11:58.481 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39604.log 2026-02-18T07:11:58.481 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50702.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50702.log.gz 2026-02-18T07:11:58.482 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28583.log 2026-02-18T07:11:58.482 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39604.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39604.log.gz 2026-02-18T07:11:58.483 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67080.log 2026-02-18T07:11:58.483 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28583.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28583.log.gz 2026-02-18T07:11:58.484 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40231.log 2026-02-18T07:11:58.484 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67080.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67080.log.gz 2026-02-18T07:11:58.484 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50678.log 2026-02-18T07:11:58.485 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40231.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40231.log.gz 2026-02-18T07:11:58.485 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55057.log 2026-02-18T07:11:58.486 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50678.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50678.log.gz 2026-02-18T07:11:58.486 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63699.log 2026-02-18T07:11:58.487 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55057.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55057.log.gz 2026-02-18T07:11:58.487 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53982.log 2026-02-18T07:11:58.488 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63699.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63699.log.gz 2026-02-18T07:11:58.488 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36723.log 2026-02-18T07:11:58.490 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53982.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53982.log.gz 2026-02-18T07:11:58.490 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45973.log 2026-02-18T07:11:58.498 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36723.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42165.log.gz 2026-02-18T07:11:58.498 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53716.log 2026-02-18T07:11:58.499 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45973.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45973.log.gz 2026-02-18T07:11:58.499 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63263.log 2026-02-18T07:11:58.499 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53716.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53716.log.gz 2026-02-18T07:11:58.500 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30007.log 2026-02-18T07:11:58.500 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63263.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63263.log.gz 2026-02-18T07:11:58.501 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45891.log 2026-02-18T07:11:58.511 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.30007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51112.log 2026-02-18T07:11:58.516 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45891.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45891.log.gz 2026-02-18T07:11:58.521 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38893.log 2026-02-18T07:11:58.521 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51112.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51112.log.gz 2026-02-18T07:11:58.531 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33645.log 2026-02-18T07:11:58.531 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38893.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38893.log.gz 2026-02-18T07:11:58.534 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40814.log.gz 2026-02-18T07:11:58.535 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60226.log 2026-02-18T07:11:58.535 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33645.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33645.log.gz 2026-02-18T07:11:58.536 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49091.log 2026-02-18T07:11:58.536 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60226.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.30007.log.gz 2026-02-18T07:11:58.537 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60226.log.gz 2026-02-18T07:11:58.537 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61013.log 2026-02-18T07:11:58.537 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35844.log 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49091.log: /var/log/ceph/cluster1-client.mirror.61013.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49091.log.gz 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61013.log.gz 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42587.log 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.35844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50943.log 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.35844.log.gz 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42587.log: 79.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42587.log.gz 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65463.log 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55515.log 2026-02-18T07:11:58.540 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50943.log.gz 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67715.log 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.55515.log: -- replaced with /var/log/ceph/cluster2-client.mirror.65463.log.gz 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55515.log.gz 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68108.log 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37186.log 2026-02-18T07:11:58.544 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.67715.log: /var/log/ceph/cluster2-client.admin.68108.log: 27.4% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68108.log.gz 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67715.log.gz 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67142.log 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35796.log 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67142.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67142.log.gz 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67662.log 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35796.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35796.log.gz 2026-02-18T07:11:58.545 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36310.log 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67662.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67662.log.gz 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54464.log 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37186.log.gz 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58310.log 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54464.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.36310.log.gz 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33666.log 2026-02-18T07:11:58.547 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58310.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54464.log.gz 2026-02-18T07:11:58.549 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58310.log.gz 2026-02-18T07:11:58.549 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53092.log 2026-02-18T07:11:58.549 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46095.log 2026-02-18T07:11:58.549 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53092.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53092.log.gz 2026-02-18T07:11:58.550 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50080.log 2026-02-18T07:11:58.550 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46095.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.46095.log.gz 2026-02-18T07:11:58.550 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48742.log 2026-02-18T07:11:58.551 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.50080.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50080.log.gz 2026-02-18T07:11:58.552 INFO:teuthology.orchestra.run.trial051.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.30113.log 2026-02-18T07:11:58.552 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.33666.log.gz 2026-02-18T07:11:58.553 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49749.log 2026-02-18T07:11:58.553 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.30113.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48742.log.gz 2026-02-18T07:11:58.553 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41450.log 2026-02-18T07:11:58.555 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.49749.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.0.30113.log.gz 2026-02-18T07:11:58.555 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30209.log 2026-02-18T07:11:58.570 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59188.log 2026-02-18T07:11:58.573 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30209.log: 95.1% -- replaced with /var/log/ceph/cluster1-client.mirror.30209.log.gz 2026-02-18T07:11:58.577 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49749.log.gz 2026-02-18T07:11:58.577 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.35439.log 2026-02-18T07:11:58.578 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59188.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59188.log.gz 2026-02-18T07:11:58.578 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54323.log 2026-02-18T07:11:58.581 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.35439.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.35439.log.gz 2026-02-18T07:11:58.581 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30113.log 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54323.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54323.log.gz 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66768.log 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30113.log: 83.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30113.log.gz 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56691.log 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66768.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66768.log.gz 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60002.log 2026-02-18T07:11:58.584 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56691.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56691.log.gz 2026-02-18T07:11:58.587 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61486.log 2026-02-18T07:11:58.587 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60002.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60002.log.gz 2026-02-18T07:11:58.587 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36659.log 2026-02-18T07:11:58.587 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61486.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61486.log.gz 2026-02-18T07:11:58.587 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48128.log 2026-02-18T07:11:58.590 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36659.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36659.log.gz 2026-02-18T07:11:58.590 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48862.log 2026-02-18T07:11:58.592 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48128.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48128.log.gz 2026-02-18T07:11:58.592 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59539.log 2026-02-18T07:11:58.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48862.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48862.log.gz 2026-02-18T07:11:58.597 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64894.log 2026-02-18T07:11:58.597 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59539.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41983.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59539.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51495.log 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59601.log 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64894.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48015.log 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59601.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59601.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51495.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61958.log 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48814.log 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48015.log: /var/log/ceph/cluster1-client.mirror.61958.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61958.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.48015.log.gz 2026-02-18T07:11:58.598 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60608.log 2026-02-18T07:11:58.599 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53644.log 2026-02-18T07:11:58.599 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48814.log: /var/log/ceph/cluster1-client.mirror.60608.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48814.log.gz 2026-02-18T07:11:58.599 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60608.log.gz 2026-02-18T07:11:58.599 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34425.log 2026-02-18T07:11:58.599 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55860.log 2026-02-18T07:11:58.601 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53644.log: /var/log/ceph/cluster1-client.mirror.34425.log: 77.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34425.log.gz 2026-02-18T07:11:58.601 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53644.log.gz 2026-02-18T07:11:58.601 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28485.log 2026-02-18T07:11:58.602 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.65797.log 2026-02-18T07:11:58.602 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55860.log.gz 2026-02-18T07:11:58.602 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38084.log 2026-02-18T07:11:58.602 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28485.log.gz/var/log/ceph/cluster2-client.mirror.0.65797.log: 2026-02-18T07:11:58.603 INFO:teuthology.orchestra.run.trial051.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.65797.log.gz 2026-02-18T07:11:58.603 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49360.log 2026-02-18T07:11:58.605 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.38084.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38084.log.gz 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57772.log 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64254.log 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49360.log: /var/log/ceph/cluster2-client.mirror.57772.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49360.log.gz 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57772.log.gz 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30621.log 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66563.log 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64254.log.gz 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30621.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30621.log.gz 2026-02-18T07:11:58.608 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31116.log 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65607.log 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66563.log: /var/log/ceph/cluster2-client.mirror.31116.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31116.log.gz 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66563.log.gz 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68353.log 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38873.log 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65607.log: /var/log/ceph/cluster2-client.admin.68353.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.68353.log.gz 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65607.log.gz 2026-02-18T07:11:58.610 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48883.log 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41073.log 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38873.log: /var/log/ceph/cluster2-client.mirror.48883.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38873.log.gz 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62478.log 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr: 92.6%/var/log/ceph/cluster2-client.mirror.41073.log: -- replaced with /var/log/ceph/cluster2-client.mirror.48883.log.gz 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41073.log.gz 2026-02-18T07:11:58.612 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48477.log 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33597.log 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62478.log: /var/log/ceph/cluster1-client.mirror.48477.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62478.log.gz 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48477.log.gz 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43143.log 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33597.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33597.log.gz 2026-02-18T07:11:58.614 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38605.log 2026-02-18T07:11:58.615 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53516.log 2026-02-18T07:11:58.615 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43143.log: /var/log/ceph/cluster2-client.mirror.38605.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38605.log.gz 2026-02-18T07:11:58.621 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62071.log 2026-02-18T07:11:58.622 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.53516.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53516.log.gz 2026-02-18T07:11:58.624 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57069.log 2026-02-18T07:11:58.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62071.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62071.log.gz 2026-02-18T07:11:58.624 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54766.log 2026-02-18T07:11:58.624 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57069.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57069.log.gz 2026-02-18T07:11:58.624 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56841.log 2026-02-18T07:11:58.626 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.54766.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54766.log.gz 2026-02-18T07:11:58.626 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41329.log 2026-02-18T07:11:58.626 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56841.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56841.log.gz 2026-02-18T07:11:58.626 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59084.log 2026-02-18T07:11:58.627 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41329.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41329.log.gz 2026-02-18T07:11:58.627 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67836.log 2026-02-18T07:11:58.628 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59084.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59084.log.gz 2026-02-18T07:11:58.629 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38972.log 2026-02-18T07:11:58.629 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.67836.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.67836.log.gz 2026-02-18T07:11:58.629 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66645.log 2026-02-18T07:11:58.630 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38972.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38972.log.gz 2026-02-18T07:11:58.631 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55836.log 2026-02-18T07:11:58.631 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66645.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66645.log.gz 2026-02-18T07:11:58.632 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63202.log 2026-02-18T07:11:58.632 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55836.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55836.log.gz 2026-02-18T07:11:58.633 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46413.log 2026-02-18T07:11:58.633 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63202.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63202.log.gz 2026-02-18T07:11:58.633 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51758.log 2026-02-18T07:11:58.634 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46413.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.46413.log.gz 2026-02-18T07:11:58.635 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31184.log 2026-02-18T07:11:58.636 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51758.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51758.log.gz 2026-02-18T07:11:58.636 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58413.log 2026-02-18T07:11:58.637 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31184.log: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.31184.log.gz 2026-02-18T07:11:58.637 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39720.log 2026-02-18T07:11:58.638 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58413.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58413.log.gz 2026-02-18T07:11:58.639 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41766.log 2026-02-18T07:11:58.639 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39720.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39720.log.gz 2026-02-18T07:11:58.640 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29612.log 2026-02-18T07:11:58.640 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41766.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41766.log.gz 2026-02-18T07:11:58.640 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41941.log 2026-02-18T07:11:58.640 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.29612.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.29612.log.gz 2026-02-18T07:11:58.641 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66457.log 2026-02-18T07:11:58.642 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41941.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41941.log.gz 2026-02-18T07:11:58.642 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39903.log 2026-02-18T07:11:58.643 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.66457.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66457.log.gz 2026-02-18T07:11:58.643 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50321.log 2026-02-18T07:11:58.643 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39903.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40061.log.gz 2026-02-18T07:11:58.644 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59725.log 2026-02-18T07:11:58.644 INFO:teuthology.orchestra.run.trial051.stderr: 93.8%/var/log/ceph/cluster2-client.mirror.50321.log: -- replaced with /var/log/ceph/cluster2-client.mirror.42251.log.gz 2026-02-18T07:11:58.645 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44292.log 2026-02-18T07:11:58.645 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59725.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50321.log.gz 2026-02-18T07:11:58.646 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59725.log.gz 2026-02-18T07:11:58.646 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58661.log 2026-02-18T07:11:58.646 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52378.log 2026-02-18T07:11:58.647 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44292.log: /var/log/ceph/cluster2-client.mirror.58661.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44292.log.gz 2026-02-18T07:11:58.647 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45435.log 2026-02-18T07:11:58.647 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52378.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58661.log.gz 2026-02-18T07:11:58.648 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45372.log 2026-02-18T07:11:58.648 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.45435.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45435.log.gz 2026-02-18T07:11:58.649 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65587.log 2026-02-18T07:11:58.649 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45372.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45372.log.gz 2026-02-18T07:11:58.650 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56044.log 2026-02-18T07:11:58.650 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65587.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65587.log.gz 2026-02-18T07:11:58.651 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56020.log 2026-02-18T07:11:58.651 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56044.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56044.log.gz 2026-02-18T07:11:58.652 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41391.log 2026-02-18T07:11:58.652 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56020.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56020.log.gz 2026-02-18T07:11:58.653 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46770.log 2026-02-18T07:11:58.653 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41391.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52378.log.gz 2026-02-18T07:11:58.654 INFO:teuthology.orchestra.run.trial051.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41391.log.gz 2026-02-18T07:11:58.654 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30570.log 2026-02-18T07:11:58.654 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46770.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57234.log 2026-02-18T07:11:58.655 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.46770.log.gz 2026-02-18T07:11:58.655 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30570.log: 62.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30570.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59663.log 2026-02-18T07:11:58.655 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.655 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57234.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61036.log 2026-02-18T07:11:58.656 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59663.log: 90.1% 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59663.log.gz 2026-02-18T07:11:58.656 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.57234.log.gz 2026-02-18T07:11:58.656 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29563.log 2026-02-18T07:11:58.656 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62951.log 2026-02-18T07:11:58.657 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61036.log: /var/log/ceph/cluster1-client.admin.29563.log: 0.0% 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61036.log.gz 2026-02-18T07:11:58.657 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.29563.log.gz 2026-02-18T07:11:58.657 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47692.log 2026-02-18T07:11:58.658 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51307.log 2026-02-18T07:11:58.658 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.62951.log: /var/log/ceph/cluster2-client.mirror.47692.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62951.log.gz 2026-02-18T07:11:58.659 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.47692.log.gz 2026-02-18T07:11:58.659 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55358.log 2026-02-18T07:11:58.660 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41507.log 2026-02-18T07:11:58.660 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.51307.log: /var/log/ceph/cluster1-client.mirror.55358.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55358.log.gz 2026-02-18T07:11:58.660 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51307.log.gz 2026-02-18T07:11:58.661 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58847.log 2026-02-18T07:11:58.661 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56820.log 2026-02-18T07:11:58.661 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41904.log.gz 2026-02-18T07:11:58.661 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41507.log.gz 2026-02-18T07:11:58.661 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40851.log 2026-02-18T07:11:58.662 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58889.log 2026-02-18T07:11:58.662 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56820.log.gz 2026-02-18T07:11:58.662 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31099.log 2026-02-18T07:11:58.662 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58847.log.gz 2026-02-18T07:11:58.663 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40851.log.gz/var/log/ceph/cluster2-client.mirror.58889.log: 2026-02-18T07:11:58.663 INFO:teuthology.orchestra.run.trial051.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67416.log 2026-02-18T07:11:58.663 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58889.log.gz 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.31099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37822.log 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31099.log.gz 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41612.log 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67416.log.gz 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38405.log 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.37822.log: /var/log/ceph/cluster2-client.mirror.41612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50750.log 2026-02-18T07:11:58.664 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38405.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38405.log.gz 2026-02-18T07:11:58.665 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45051.log 2026-02-18T07:11:58.665 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50750.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50750.log.gz 2026-02-18T07:11:58.665 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62708.log 2026-02-18T07:11:58.666 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45051.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45051.log.gz 2026-02-18T07:11:58.666 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37498.log 2026-02-18T07:11:58.666 INFO:teuthology.orchestra.run.trial051.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37822.log.gz 2026-02-18T07:11:58.667 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.62708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44200.log 2026-02-18T07:11:58.667 INFO:teuthology.orchestra.run.trial051.stderr: 86.4%/var/log/ceph/cluster1-client.mirror.37498.log: -- replaced with /var/log/ceph/cluster1-client.mirror.62708.log.gz 2026-02-18T07:11:58.668 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66013.log 2026-02-18T07:11:58.669 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44200.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44200.log.gz 2026-02-18T07:11:58.670 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46991.log 2026-02-18T07:11:58.670 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66013.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37498.log.gz 2026-02-18T07:11:58.670 INFO:teuthology.orchestra.run.trial051.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66013.log.gz 2026-02-18T07:11:58.671 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36750.log 2026-02-18T07:11:58.671 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46991.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66978.log 2026-02-18T07:11:58.671 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46991.log.gz 2026-02-18T07:11:58.671 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.36750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48969.log 2026-02-18T07:11:58.672 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66978.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.36750.log.gz 2026-02-18T07:11:58.672 INFO:teuthology.orchestra.run.trial051.stderr: 93.9% 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.66978.log.gz 2026-02-18T07:11:58.672 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.41612.log.gz 2026-02-18T07:11:58.673 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39421.log 2026-02-18T07:11:58.673 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29367.log 2026-02-18T07:11:58.673 INFO:teuthology.orchestra.run.trial051.stderr: 89.1%/var/log/ceph/cluster2-client.mirror.39421.log: -- replaced with /var/log/ceph/cluster2-client.mirror.48969.log.gz 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43605.log 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:gzip 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39421.log.gz 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46475.log 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.29367.log: /var/log/ceph/cluster2-client.mirror.43605.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.29367.log.gz 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46160.log 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43605.log.gz 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45597.log 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46475.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.33316.log 2026-02-18T07:11:58.674 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.46160.log: /var/log/ceph/cluster1-client.mirror.45597.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.46475.log.gz 2026-02-18T07:11:58.675 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45597.log.gz 2026-02-18T07:11:58.675 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36843.log 2026-02-18T07:11:58.675 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.3.33316.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60271.log 2026-02-18T07:11:58.675 INFO:teuthology.orchestra.run.trial051.stderr: 92.4%/var/log/ceph/cluster2-client.mirror.36843.log: -- replaced with /var/log/ceph/cluster1-client.mirror.46160.log.gz 2026-02-18T07:11:58.676 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36843.log.gz 2026-02-18T07:11:58.676 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55788.log 2026-02-18T07:11:58.676 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35091.log 2026-02-18T07:11:58.677 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60271.log.gz 2026-02-18T07:11:58.677 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54643.log 2026-02-18T07:11:58.678 INFO:teuthology.orchestra.run.trial051.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.33316.log.gz 2026-02-18T07:11:58.678 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35091.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55788.log.gz 2026-02-18T07:11:58.678 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43443.log 2026-02-18T07:11:58.678 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32606.log 2026-02-18T07:11:58.678 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54643.log.gz 2026-02-18T07:11:58.679 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63009.log 2026-02-18T07:11:58.680 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.32606.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35091.log.gz 2026-02-18T07:11:58.681 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60203.log 2026-02-18T07:11:58.681 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63009.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63009.log.gz 2026-02-18T07:11:58.682 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63637.log 2026-02-18T07:11:58.682 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60203.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32606.log.gz 2026-02-18T07:11:58.682 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60203.log.gz 2026-02-18T07:11:58.683 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64935.log 2026-02-18T07:11:58.683 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37102.log 2026-02-18T07:11:58.683 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63637.log.gz 2026-02-18T07:11:58.684 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53961.log 2026-02-18T07:11:58.684 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64935.log.gz 2026-02-18T07:11:58.684 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.37102.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47541.log 2026-02-18T07:11:58.684 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53961.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.37102.log.gz 2026-02-18T07:11:58.685 INFO:teuthology.orchestra.run.trial051.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53961.log.gz 2026-02-18T07:11:58.685 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43627.log 2026-02-18T07:11:58.685 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56569.log 2026-02-18T07:11:58.685 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47541.log.gz 2026-02-18T07:11:58.686 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43627.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50252.log 2026-02-18T07:11:58.686 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56569.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56569.log.gz 2026-02-18T07:11:58.687 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51695.log 2026-02-18T07:11:58.687 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.50252.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50252.log.gz 2026-02-18T07:11:58.687 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39802.log 2026-02-18T07:11:58.688 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51695.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51695.log.gz 2026-02-18T07:11:58.688 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39061.log 2026-02-18T07:11:58.689 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39802.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39802.log.gz 2026-02-18T07:11:58.690 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42728.log 2026-02-18T07:11:58.690 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42545.log.gz 2026-02-18T07:11:58.690 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39061.log: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48150.log 2026-02-18T07:11:58.690 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39061.log.gz 2026-02-18T07:11:58.691 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42728.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36636.log 2026-02-18T07:11:58.691 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42728.log.gz 2026-02-18T07:11:58.691 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65566.log 2026-02-18T07:11:58.691 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36636.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48150.log.gz 2026-02-18T07:11:58.692 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58744.log 2026-02-18T07:11:58.692 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65566.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65566.log.gz 2026-02-18T07:11:58.693 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67206.log 2026-02-18T07:11:58.693 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58744.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58744.log.gz 2026-02-18T07:11:58.694 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33107.log 2026-02-18T07:11:58.694 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67206.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.36636.log.gz 2026-02-18T07:11:58.695 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63345.log 2026-02-18T07:11:58.695 INFO:teuthology.orchestra.run.trial051.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67206.log.gz 2026-02-18T07:11:58.695 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.33107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66522.log 2026-02-18T07:11:58.696 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63345.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63345.log.gz 2026-02-18T07:11:58.696 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43699.log 2026-02-18T07:11:58.697 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66522.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66522.log.gz 2026-02-18T07:11:58.697 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59952.log 2026-02-18T07:11:58.698 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43699.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43699.log.gz 2026-02-18T07:11:58.698 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41169.log 2026-02-18T07:11:58.698 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59952.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40340.log.gz 2026-02-18T07:11:58.699 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30723.log 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59952.log.gz 2026-02-18T07:11:58.699 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.699 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41169.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.33107.log.gz 2026-02-18T07:11:58.700 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51647.log 2026-02-18T07:11:58.700 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.30723.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.30723.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56430.log 2026-02-18T07:11:58.700 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41169.log.gz 2026-02-18T07:11:58.700 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51647.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38385.log 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51647.log.gz 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.56430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41231.log 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56430.log.gz/var/log/ceph/cluster2-client.mirror.38385.log: 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44537.log 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.38385.log.gz 2026-02-18T07:11:58.701 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54816.log 2026-02-18T07:11:58.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41231.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41231.log.gz 2026-02-18T07:11:58.702 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46720.log 2026-02-18T07:11:58.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.44537.log: /var/log/ceph/cluster1-client.mirror.54816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36268.log 2026-02-18T07:11:58.702 INFO:teuthology.orchestra.run.trial051.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54816.log.gz 2026-02-18T07:11:58.702 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.46720.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44537.log.gz 2026-02-18T07:11:58.703 INFO:teuthology.orchestra.run.trial051.stderr:gzip 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.46720.log.gz 2026-02-18T07:11:58.703 INFO:teuthology.orchestra.run.trial051.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53029.log 2026-02-18T07:11:58.703 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36268.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33549.log 2026-02-18T07:11:58.703 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53029.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33729.log 2026-02-18T07:11:58.703 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53029.log.gz 2026-02-18T07:11:58.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60856.log 2026-02-18T07:11:58.704 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.33729.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.33549.log.gz 2026-02-18T07:11:58.705 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44585.log 2026-02-18T07:11:58.705 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60856.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60856.log.gz 2026-02-18T07:11:58.705 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39663.log 2026-02-18T07:11:58.706 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44585.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36268.log.gz 2026-02-18T07:11:58.706 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.44585.log.gz 2026-02-18T07:11:58.706 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65230.log 2026-02-18T07:11:58.707 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39663.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.33729.log.gz 2026-02-18T07:11:58.708 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54371.log 2026-02-18T07:11:58.708 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34158.log 2026-02-18T07:11:58.709 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54371.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54371.log.gz 2026-02-18T07:11:58.709 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61463.log 2026-02-18T07:11:58.711 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34158.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36332.log.gz 2026-02-18T07:11:58.713 INFO:teuthology.orchestra.run.trial051.stderr: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42106.log 2026-02-18T07:11:58.713 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.34158.log.gz 2026-02-18T07:11:58.713 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38322.log 2026-02-18T07:11:58.713 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42106.log: 86.7% 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42106.log.gz 2026-02-18T07:11:58.713 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.61463.log.gz 2026-02-18T07:11:58.714 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66998.log 2026-02-18T07:11:58.717 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.38322.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38322.log.gz 2026-02-18T07:11:58.717 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59064.log 2026-02-18T07:11:58.718 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66998.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66998.log.gz 2026-02-18T07:11:58.718 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57214.log 2026-02-18T07:11:58.719 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.30209.log 2026-02-18T07:11:58.719 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59064.log: /var/log/ceph/cluster2-client.mirror.57214.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59064.log.gz 2026-02-18T07:11:58.719 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57214.log.gz 2026-02-18T07:11:58.720 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59911.log 2026-02-18T07:11:58.726 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.2.30209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.35949.log 2026-02-18T07:11:58.726 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59911.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59911.log.gz 2026-02-18T07:11:58.727 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34566.log 2026-02-18T07:11:58.731 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.1.35949.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.35949.log.gz 2026-02-18T07:11:58.731 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51357.log 2026-02-18T07:11:58.732 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.34566.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.34566.log.gz 2026-02-18T07:11:58.732 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58144.log 2026-02-18T07:11:58.733 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51357.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51357.log.gz 2026-02-18T07:11:58.733 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63719.log 2026-02-18T07:11:58.734 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58144.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58144.log.gz 2026-02-18T07:11:58.734 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55310.log 2026-02-18T07:11:58.735 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63719.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63719.log.gz 2026-02-18T07:11:58.735 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32585.log 2026-02-18T07:11:58.736 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.55310.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55310.log.gz 2026-02-18T07:11:58.736 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44915.log 2026-02-18T07:11:58.737 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32585.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.32585.log.gz 2026-02-18T07:11:58.737 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49115.log 2026-02-18T07:11:58.739 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.44915.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44915.log.gz 2026-02-18T07:11:58.739 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61148.log 2026-02-18T07:11:58.740 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49115.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49115.log.gz 2026-02-18T07:11:58.740 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48208.log 2026-02-18T07:11:58.741 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61148.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61148.log.gz 2026-02-18T07:11:58.741 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35251.log 2026-02-18T07:11:58.742 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48208.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48208.log.gz 2026-02-18T07:11:58.742 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42123.log 2026-02-18T07:11:58.746 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35251.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41052.log.gz 2026-02-18T07:11:58.751 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66095.log 2026-02-18T07:11:58.752 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42123.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42123.log.gz 2026-02-18T07:11:58.752 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55033.log 2026-02-18T07:11:58.753 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66095.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.66095.log.gz 2026-02-18T07:11:58.753 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43472.log 2026-02-18T07:11:58.754 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.55033.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55033.log.gz 2026-02-18T07:11:58.754 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54484.log 2026-02-18T07:11:58.755 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43472.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43472.log.gz 2026-02-18T07:11:58.755 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32334.log 2026-02-18T07:11:58.756 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54484.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54484.log.gz 2026-02-18T07:11:58.757 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39183.log 2026-02-18T07:11:58.759 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.32334.log: 95.5% -- replaced with /var/log/ceph/cluster1-client.mirror.32334.log.gz 2026-02-18T07:11:58.760 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33410.log 2026-02-18T07:11:58.769 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67457.log 2026-02-18T07:11:58.775 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.33410.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33773.log.gz 2026-02-18T07:11:58.776 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66501.log 2026-02-18T07:11:58.776 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.67457.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67457.log.gz 2026-02-18T07:11:58.777 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42433.log.gz 2026-02-18T07:11:58.777 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31048.log 2026-02-18T07:11:58.777 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.66501.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.33410.log.gz 2026-02-18T07:11:58.778 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66501.log.gz 2026-02-18T07:11:58.778 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54864.log 2026-02-18T07:11:58.778 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.31048.log: 80.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63072.log 2026-02-18T07:11:58.778 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.31048.log.gz 2026-02-18T07:11:58.779 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.54864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.62708.log 2026-02-18T07:11:58.779 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63072.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54864.log.gz 2026-02-18T07:11:58.779 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63072.log.gz 2026-02-18T07:11:58.780 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55539.log 2026-02-18T07:11:58.780 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56525.log 2026-02-18T07:11:58.780 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.0.62708.log: /var/log/ceph/cluster1-client.mirror.55539.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55539.log.gz 2026-02-18T07:11:58.781 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40772.log 2026-02-18T07:11:58.781 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.56525.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56525.log.gz 2026-02-18T07:11:58.782 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28632.log 2026-02-18T07:11:58.782 INFO:teuthology.orchestra.run.trial051.stderr: 93.0% -- replaced with /var/log/ceph/cluster2-client.mirror.0.62708.log.gz 2026-02-18T07:11:58.783 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59518.log 2026-02-18T07:11:58.783 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.admin.28632.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.28632.log.gz 2026-02-18T07:11:58.783 INFO:teuthology.orchestra.run.trial051.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40772.log.gz 2026-02-18T07:11:58.783 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52956.log 2026-02-18T07:11:58.784 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.59518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41723.log 2026-02-18T07:11:58.784 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52956.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52956.log.gz 2026-02-18T07:11:58.784 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49264.log 2026-02-18T07:11:58.784 INFO:teuthology.orchestra.run.trial051.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59518.log.gz 2026-02-18T07:11:58.785 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52330.log 2026-02-18T07:11:58.785 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.49264.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49264.log.gz 2026-02-18T07:11:58.786 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63386.log 2026-02-18T07:11:58.786 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.52330.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52330.log.gz 2026-02-18T07:11:58.787 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61621.log 2026-02-18T07:11:58.787 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63386.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63386.log.gz 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr: 93.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30495.log 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.30209.log.gz 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61621.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60451.log 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.admin.30495.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61621.log.gz 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.30495.log.gz 2026-02-18T07:11:58.789 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45475.log 2026-02-18T07:11:58.790 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57668.log 2026-02-18T07:11:58.790 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60451.log: /var/log/ceph/cluster2-client.mirror.45475.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60451.log.gz 2026-02-18T07:11:58.790 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61351.log 2026-02-18T07:11:58.790 INFO:teuthology.orchestra.run.trial051.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45475.log.gz 2026-02-18T07:11:58.791 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.57668.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57668.log.gz 2026-02-18T07:11:58.791 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35897.log 2026-02-18T07:11:58.791 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43682.log 2026-02-18T07:11:58.792 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61351.log: /var/log/ceph/cluster2-client.mirror.35897.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61351.log.gz 2026-02-18T07:11:58.792 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61261.log 2026-02-18T07:11:58.792 INFO:teuthology.orchestra.run.trial051.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35897.log.gz 2026-02-18T07:11:58.792 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43682.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43682.log.gz 2026-02-18T07:11:58.792 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42391.log 2026-02-18T07:11:58.793 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41211.log 2026-02-18T07:11:58.793 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61261.log: /var/log/ceph/cluster2-client.mirror.42391.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61261.log.gz 2026-02-18T07:11:58.793 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53447.log 2026-02-18T07:11:58.794 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42391.log.gz 2026-02-18T07:11:58.795 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41211.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66370.log.gz 2026-02-18T07:11:58.804 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36084.log 2026-02-18T07:11:58.804 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.53447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35570.log 2026-02-18T07:11:58.804 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.36084.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53447.log.gz 2026-02-18T07:11:58.805 INFO:teuthology.orchestra.run.trial051.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36084.log.gz 2026-02-18T07:11:58.817 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51208.log 2026-02-18T07:11:58.817 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.35570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30160.log 2026-02-18T07:11:58.817 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.51208.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35570.log.gz 2026-02-18T07:11:58.818 INFO:teuthology.orchestra.run.trial051.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51208.log.gz 2026-02-18T07:11:58.822 INFO:teuthology.orchestra.run.trial051.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35251.log.gz 2026-02-18T07:11:58.823 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43568.log 2026-02-18T07:11:58.824 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.30160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65360.log 2026-02-18T07:11:58.824 INFO:teuthology.orchestra.run.trial051.stderr: 83.7% -- replaced with /var/log/ceph/cluster1-client.mirror.30160.log.gz 2026-02-18T07:11:58.824 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.43568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64524.log 2026-02-18T07:11:58.824 INFO:teuthology.orchestra.run.trial051.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43568.log.gz 2026-02-18T07:11:58.824 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.65360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55192.log 2026-02-18T07:11:58.825 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65360.log.gz 2026-02-18T07:11:58.825 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61756.log 2026-02-18T07:11:58.825 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64524.log: /var/log/ceph/cluster2-client.mirror.55192.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64524.log.gz 2026-02-18T07:11:58.825 INFO:teuthology.orchestra.run.trial051.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55192.log.gz 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45798.log 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57730.log 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61756.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61756.log.gz 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47589.log 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.45798.log: /var/log/ceph/cluster2-client.mirror.57730.log: 90.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64852.log 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.45798.log.gz 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.47589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39641.log 2026-02-18T07:11:58.830 INFO:teuthology.orchestra.run.trial051.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47589.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.64852.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57730.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48766.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64852.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55105.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.39641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63160.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39641.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.48766.log: /var/log/ceph/cluster1-client.mirror.55105.log: gzip 87.4% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.48766.log.gz --verbose 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr: -- /var/log/ceph/cluster2-client.mirror.46929.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55105.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.63160.log: /var/log/ceph/cluster2-client.mirror.46929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48349.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.63160.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-osd.2.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46929.log.gz 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36156.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.48349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64463.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52536.log 2026-02-18T07:11:58.831 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36156.log: /var/log/ceph/cluster2-client.mirror.64463.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36156.log.gz 2026-02-18T07:11:58.832 INFO:teuthology.orchestra.run.trial051.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64463.log.gz 2026-02-18T07:11:58.832 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48349.log.gz 2026-02-18T07:11:58.832 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61598.log 2026-02-18T07:11:58.832 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60046.log 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.52536.log: /var/log/ceph/cluster1-client.mirror.61598.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63887.log 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61598.log.gz 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41010.log 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.63887.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60046.log.gz 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr: 82.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63887.log.gz 2026-02-18T07:11:58.836 INFO:teuthology.orchestra.run.trial051.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52536.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42503.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42374.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.41010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61328.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42503.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41010.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.42374.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42374.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42503.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39315.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60676.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.61328.log: /var/log/ceph/cluster2-client.mirror.39315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58061.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61328.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.60676.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40930.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.60676.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.58061.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36595.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster2-client.mirror.40930.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58061.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40930.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67264.log 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr:/var/log/ceph/cluster1-client.mirror.36595.log: /var/log/ceph/cluster2-client.mirror.67264.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.36595.log.gz 2026-02-18T07:11:58.837 INFO:teuthology.orchestra.run.trial051.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67264.log.gz 2026-02-18T07:11:58.847 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41450.log.gz 2026-02-18T07:11:58.849 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39315.log.gz 2026-02-18T07:11:58.852 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39271.log.gz 2026-02-18T07:11:58.855 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63507.log.gz 2026-02-18T07:11:58.930 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35198.log.gz 2026-02-18T07:11:58.954 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43443.log.gz 2026-02-18T07:11:58.988 INFO:teuthology.orchestra.run.trial051.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43627.log.gz 2026-02-18T07:11:58.991 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42646.log.gz 2026-02-18T07:11:59.009 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43143.log.gz 2026-02-18T07:11:59.015 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39663.log.gz 2026-02-18T07:11:59.035 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42960.log.gz 2026-02-18T07:11:59.064 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67286.log.gz 2026-02-18T07:11:59.065 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39903.log.gz 2026-02-18T07:11:59.112 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65230.log.gz 2026-02-18T07:11:59.168 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36723.log.gz 2026-02-18T07:11:59.178 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41211.log.gz 2026-02-18T07:11:59.192 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41723.log.gz 2026-02-18T07:11:59.200 INFO:teuthology.orchestra.run.trial051.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33171.log.gz 2026-02-18T07:11:59.324 INFO:teuthology.orchestra.run.trial051.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39183.log.gz 2026-02-18T07:12:01.313 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.mon.a has been restored 2026-02-18T07:12:01.822 INFO:teuthology.orchestra.run.trial006.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-18T07:12:02.196 INFO:teuthology.orchestra.run.trial006.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-18T07:12:02.362 INFO:teuthology.orchestra.run.trial006.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-18T07:12:02.364 INFO:teuthology.orchestra.run.trial006.stderr: 2026-02-18T07:12:02.364 INFO:teuthology.orchestra.run.trial006.stderr:real 0m5.146s 2026-02-18T07:12:02.364 INFO:teuthology.orchestra.run.trial006.stderr:user 0m14.731s 2026-02-18T07:12:02.364 INFO:teuthology.orchestra.run.trial006.stderr:sys 0m0.791s 2026-02-18T07:13:07.422 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-18T07:13:09.797 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-18T07:13:33.254 INFO:teuthology.orchestra.run.trial051.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-18T07:13:33.257 INFO:teuthology.orchestra.run.trial051.stderr: 2026-02-18T07:13:33.257 INFO:teuthology.orchestra.run.trial051.stderr:real 1m36.013s 2026-02-18T07:13:33.257 INFO:teuthology.orchestra.run.trial051.stderr:user 4m20.551s 2026-02-18T07:13:33.257 INFO:teuthology.orchestra.run.trial051.stderr:sys 0m12.978s 2026-02-18T07:13:33.257 INFO:tasks.ceph:Archiving logs... 2026-02-18T07:13:33.258 DEBUG:teuthology.misc:Transferring archived files from trial006:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial006/log 2026-02-18T07:13:33.259 DEBUG:teuthology.orchestra.run.trial006:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-18T07:13:34.385 DEBUG:teuthology.misc:Transferring archived files from trial051:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial051/log 2026-02-18T07:13:34.386 DEBUG:teuthology.orchestra.run.trial051:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-18T07:13:54.106 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-18T07:13:54.117 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-18T07:13:54.118 DEBUG:teuthology.orchestra.run.trial006:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-18T07:13:54.120 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-18T07:13:54.270 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-18T07:13:54.281 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-18T07:13:54.282 DEBUG:teuthology.orchestra.run.trial006:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T07:13:54.283 DEBUG:teuthology.orchestra.run.trial051:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-18T07:13:54.297 INFO:teuthology.orchestra.run.trial006.stderr:bash: line 1: ntpq: command not found 2026-02-18T07:13:54.300 INFO:teuthology.orchestra.run.trial051.stderr:bash: line 1: ntpq: command not found 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:=============================================================================== 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:^- 23.186.168.131 2 7 377 86 +719us[ +712us] +/- 55ms 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:^- lax1.us.ntp.li 2 8 377 25 +342us[ +335us] +/- 35ms 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:^- 172-104-28-175.ip.linode> 2 7 377 84 -5468us[-5474us] +/- 57ms 2026-02-18T07:13:54.313 INFO:teuthology.orchestra.run.trial006.stdout:^* rn-02.koehn.com 2 7 377 23 -41us[ -48us] +/- 11ms 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:=============================================================================== 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:^+ time-usw1.crimpac.net 2 7 377 18 -160us[ -160us] +/- 81ms 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:^+ dns-e.ns4v.icu 2 6 377 21 -189us[ -189us] +/- 79ms 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:^* arm1.maxhost.io 2 7 377 86 +136us[ +157us] +/- 51ms 2026-02-18T07:13:54.321 INFO:teuthology.orchestra.run.trial051.stdout:^+ hou1.us.ntp.li 2 7 377 87 -203us[ -182us] +/- 71ms 2026-02-18T07:13:54.321 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-18T07:13:54.332 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-18T07:13:54.333 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-18T07:13:54.346 DEBUG:teuthology.orchestra.run.trial006:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-18T07:13:54.395 DEBUG:teuthology.orchestra.run.trial051:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-18T07:13:54.441 DEBUG:teuthology.orchestra.run.trial006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-18T07:13:54.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T07:13:54.468 DEBUG:teuthology.orchestra.run.trial051:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-18T07:13:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T07:13:54.492 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-18T07:13:54.513 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-18T07:13:54.533 INFO:teuthology.task.internal:Duration was 2490.842792 seconds 2026-02-18T07:13:54.534 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-18T07:13:54.554 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-18T07:13:54.554 DEBUG:teuthology.orchestra.run.trial006:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-18T07:13:54.557 DEBUG:teuthology.orchestra.run.trial051:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-18T07:13:54.592 INFO:teuthology.orchestra.run.trial006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-18T07:13:54.594 INFO:teuthology.orchestra.run.trial051.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-18T07:13:54.942 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-18T07:13:54.943 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial006.front.sepia.ceph.com 2026-02-18T07:13:54.943 DEBUG:teuthology.orchestra.run.trial006:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-18T07:13:55.002 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial051.front.sepia.ceph.com 2026-02-18T07:13:55.002 DEBUG:teuthology.orchestra.run.trial051:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-18T07:13:55.022 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-18T07:13:55.023 DEBUG:teuthology.orchestra.run.trial006:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T07:13:55.045 DEBUG:teuthology.orchestra.run.trial051:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T07:13:55.105 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-18T07:13:55.106 DEBUG:teuthology.orchestra.run.trial006:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T07:13:55.127 DEBUG:teuthology.orchestra.run.trial051:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-18T07:13:55.149 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T07:13:55.149 INFO:teuthology.orchestra.run.trial006.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T07:13:55.149 INFO:teuthology.orchestra.run.trial006.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T07:13:55.149 INFO:teuthology.orchestra.run.trial006.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-18T07:13:55.149 INFO:teuthology.orchestra.run.trial006.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-18T07:13:55.152 INFO:teuthology.orchestra.run.trial006.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 88.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-18T07:13:55.168 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-18T07:13:55.169 INFO:teuthology.orchestra.run.trial051.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-18T07:13:55.169 INFO:teuthology.orchestra.run.trial051.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip 0.0% -5 -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz --verbose 2026-02-18T07:13:55.169 INFO:teuthology.orchestra.run.trial051.stderr: -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-18T07:13:55.169 INFO:teuthology.orchestra.run.trial051.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-18T07:13:55.174 INFO:teuthology.orchestra.run.trial051.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 89.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-18T07:13:55.176 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-18T07:13:55.205 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-18T07:13:55.205 DEBUG:teuthology.orchestra.run.trial006:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-18T07:13:55.229 DEBUG:teuthology.orchestra.run.trial051:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-18T07:13:55.253 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-18T07:13:55.274 DEBUG:teuthology.orchestra.run.trial006:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-18T07:13:55.277 DEBUG:teuthology.orchestra.run.trial051:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-18T07:13:55.300 INFO:teuthology.orchestra.run.trial006.stdout:kernel.core_pattern = core 2026-02-18T07:13:55.316 INFO:teuthology.orchestra.run.trial051.stdout:kernel.core_pattern = core 2026-02-18T07:13:55.331 DEBUG:teuthology.orchestra.run.trial006:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-18T07:13:55.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T07:13:55.369 DEBUG:teuthology.orchestra.run.trial051:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-18T07:13:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-18T07:13:55.388 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-18T07:13:55.410 INFO:teuthology.task.internal:Transferring archived files... 2026-02-18T07:13:55.411 DEBUG:teuthology.misc:Transferring archived files from trial006:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial006 2026-02-18T07:13:55.412 DEBUG:teuthology.orchestra.run.trial006:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-18T07:13:55.446 DEBUG:teuthology.misc:Transferring archived files from trial051:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-17_21:26:40-rbd-20.2.1_2.17.26-baseline_2-distro-default-trial/54909/remote/trial051 2026-02-18T07:13:55.446 DEBUG:teuthology.orchestra.run.trial051:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-18T07:13:55.478 INFO:teuthology.task.internal:Removing archive directory... 2026-02-18T07:13:55.478 DEBUG:teuthology.orchestra.run.trial006:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-18T07:13:55.481 DEBUG:teuthology.orchestra.run.trial051:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-18T07:13:55.528 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-18T07:13:55.552 INFO:teuthology.task.internal:Not uploading archives. 2026-02-18T07:13:55.552 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-18T07:13:55.574 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-18T07:13:55.574 DEBUG:teuthology.orchestra.run.trial006:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-18T07:13:55.577 DEBUG:teuthology.orchestra.run.trial051:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-18T07:13:55.592 INFO:teuthology.orchestra.run.trial006.stdout: 791912 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 18 07:13 /home/ubuntu/cephtest 2026-02-18T07:13:55.593 INFO:teuthology.orchestra.run.trial051.stdout: 791912 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 18 07:13 /home/ubuntu/cephtest 2026-02-18T07:13:55.595 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-18T07:13:55.616 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-18T07:13:55.660 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-zlib policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-minimum} duration: 2490.842791557312 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-18T07:13:55.661 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-18T07:13:55.772 INFO:teuthology.run:pass